workflowval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. workflowval/__init__.py +3 -0
  2. workflowval/cli.py +584 -0
  3. workflowval/gen/__init__.py +194 -0
  4. workflowval/gen/source.py +109 -0
  5. workflowval/gen/style.md +17 -0
  6. workflowval/gen/templates/.gitkeep +0 -0
  7. workflowval/gen/templates/conftest.py.tmpl +332 -0
  8. workflowval/loop/__init__.py +55 -0
  9. workflowval/loop/runner.py +488 -0
  10. workflowval/loop/stats.py +178 -0
  11. workflowval/loop/trace.py +29 -0
  12. workflowval/models/__init__.py +19 -0
  13. workflowval/models/base.py +68 -0
  14. workflowval/models/fake.py +55 -0
  15. workflowval/models/litellm_adapter.py +153 -0
  16. workflowval/pipeline.py +906 -0
  17. workflowval/planner/__init__.py +62 -0
  18. workflowval/prompts/__init__.py +34 -0
  19. workflowval/prompts/planner.md +37 -0
  20. workflowval/prompts/reviewer.md +30 -0
  21. workflowval/prompts/task_answer_review.md +14 -0
  22. workflowval/prompts/task_finding.md +17 -0
  23. workflowval/prompts/task_fix_test.md +6 -0
  24. workflowval/prompts/task_fix_workflow.md +7 -0
  25. workflowval/prompts/task_patch.md +9 -0
  26. workflowval/prompts/task_plan.md +13 -0
  27. workflowval/prompts/task_plan_gaps.md +6 -0
  28. workflowval/prompts/task_report.md +9 -0
  29. workflowval/prompts/task_review_plan.md +17 -0
  30. workflowval/prompts/task_review_results.md +19 -0
  31. workflowval/prompts/task_triage.md +9 -0
  32. workflowval/prompts/writer.md +11 -0
  33. workflowval/py.typed +0 -0
  34. workflowval/redact.py +97 -0
  35. workflowval/report/__init__.py +162 -0
  36. workflowval/report/curl.py +75 -0
  37. workflowval/schemas/__init__.py +78 -0
  38. workflowval/schemas/_base.py +43 -0
  39. workflowval/schemas/auth.py +66 -0
  40. workflowval/schemas/labels.py +31 -0
  41. workflowval/schemas/results.py +222 -0
  42. workflowval/schemas/review.py +94 -0
  43. workflowval/schemas/state.py +32 -0
  44. workflowval/schemas/stats.py +36 -0
  45. workflowval/schemas/workflow.py +148 -0
  46. workflowval/spec/__init__.py +38 -0
  47. workflowval/spec/graph.py +196 -0
  48. workflowval/spec/loader.py +102 -0
  49. workflowval/spec/validation.py +98 -0
  50. workflowval/testrun.py +159 -0
  51. workflowval/tools/__init__.py +6 -0
  52. workflowval/tools/base.py +49 -0
  53. workflowval/tools/registry.py +637 -0
  54. workflowval/triage/__init__.py +199 -0
  55. workflowval-0.1.0.dist-info/METADATA +215 -0
  56. workflowval-0.1.0.dist-info/RECORD +59 -0
  57. workflowval-0.1.0.dist-info/WHEEL +4 -0
  58. workflowval-0.1.0.dist-info/entry_points.txt +2 -0
  59. workflowval-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,637 @@
1
+ """Every tool and which role may call it.
2
+
3
+ The first 12 tools are the doc's list. `read_tests` and `write_verdict` were added in the M0 review
4
+ (docs/m0-review.md). Tools that raise "arrives in" are stubs until that milestone.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import json
10
+ import re
11
+ from collections.abc import Callable
12
+ from pathlib import Path
13
+ from typing import Any, Literal
14
+
15
+ import jsonpatch
16
+ from jsonpointer import JsonPointerException, resolve_pointer
17
+ from pydantic import BaseModel, Field, ValidationError, model_validator
18
+
19
+ from workflowval.gen import (
20
+ SourceError,
21
+ check_test_source,
22
+ format_source,
23
+ hash_line,
24
+ lint_source,
25
+ test_path,
26
+ )
27
+ from workflowval.redact import redact_text
28
+ from workflowval.report.curl import curl_steps
29
+ from workflowval.schemas import (
30
+ Answer,
31
+ Basis,
32
+ Confidence,
33
+ Decision,
34
+ Label,
35
+ Review,
36
+ ReviewLog,
37
+ Source,
38
+ Workflow,
39
+ atomic_write_text,
40
+ )
41
+ from workflowval.spec import ResponseChecker, SpecGraph
42
+ from workflowval.tools.base import Tool, ToolContext, ToolError
43
+
44
+
45
+ class ReadSpecArgs(BaseModel):
46
+ """Arguments for read_spec."""
47
+
48
+ operation_id: str | None = Field(default=None, description="Omit for the whole graph")
49
+ pointer: str | None = Field(
50
+ default=None,
51
+ description="A JSON Pointer into the spec as written, with its $refs, for a spec patch",
52
+ )
53
+
54
+
55
+ class ReadDocsArgs(BaseModel):
56
+ """Arguments for read_docs."""
57
+
58
+ path: str | None = Field(default=None, description="Omit to list the docs")
59
+
60
+
61
+ class ReadWorkflowsArgs(BaseModel):
62
+ """Arguments for read_workflows."""
63
+
64
+ workflow_id: str | None = Field(default=None, description="Omit for all workflows")
65
+
66
+
67
+ class WriteWorkflowArgs(BaseModel):
68
+ """Arguments for write_workflow."""
69
+
70
+ workflow: Workflow
71
+
72
+
73
+ class WriteReviewArgs(BaseModel):
74
+ """Arguments for write_review."""
75
+
76
+ review: Review
77
+
78
+
79
+ class AnswerReviewArgs(BaseModel):
80
+ """Arguments for answer_review."""
81
+
82
+ item_id: str = Field(min_length=1)
83
+ answer: Answer
84
+
85
+
86
+ class WriteTestArgs(BaseModel):
87
+ """Arguments for write_test."""
88
+
89
+ workflow_id: str = Field(pattern=r"^[a-z0-9_]+$")
90
+ source: str = Field(min_length=1, description="The full Python source")
91
+
92
+
93
+ class ReadTestsArgs(BaseModel):
94
+ """Arguments for read_tests."""
95
+
96
+ workflow_id: str = Field(pattern=r"^[a-z0-9_]+$")
97
+
98
+
99
+ class RunTestsArgs(BaseModel):
100
+ """Arguments for run_tests."""
101
+
102
+ test_ids: list[str] = Field(default_factory=list, description="Empty runs every test")
103
+
104
+
105
+ class ReadResultsArgs(BaseModel):
106
+ """Arguments for read_results."""
107
+
108
+ test_id: str | None = Field(default=None, description="Omit for all results")
109
+
110
+
111
+ class HttpProbeArgs(BaseModel):
112
+ """Arguments for http_probe."""
113
+
114
+ method: Literal["GET", "HEAD", "OPTIONS", "POST", "PUT", "PATCH", "DELETE"]
115
+ url: str = Field(min_length=1)
116
+ headers: dict[str, str] = Field(default_factory=dict)
117
+ body: Any = None
118
+
119
+
120
+ class WriteVerdictArgs(BaseModel):
121
+ """Arguments for write_verdict."""
122
+
123
+ test_id: str = Field(min_length=1)
124
+ label: Label
125
+ confidence: Confidence
126
+ reason: str = Field(min_length=1)
127
+ fix_target: Literal["test", "workflow"] | None = Field(
128
+ default=None,
129
+ description="For TEST_BUG only: 'workflow' if an expectation or data is wrong, else 'test'",
130
+ )
131
+
132
+
133
+ class WriteBugReportArgs(BaseModel):
134
+ """Arguments for write_bug_report. Code adds the curl steps from what was recorded."""
135
+
136
+ kind: Literal["api_bug", "conflict"]
137
+ title: str = Field(min_length=1)
138
+ test_id: str = Field(min_length=1, description="The test or finding id this report covers")
139
+ expected: str = Field(min_length=1, description="For a conflict: per spec, and per docs")
140
+ actual: str = Field(min_length=1)
141
+ spec_reference: str = Field(min_length=1)
142
+ docs_excerpt: str | None = Field(default=None, description="Required for a conflict")
143
+ affected_workflows: list[str] = Field(
144
+ default_factory=list, description="Required for a conflict"
145
+ )
146
+ suggested_resolution: str | None = Field(default=None, description="Required for a conflict")
147
+
148
+ @model_validator(mode="after")
149
+ def _conflict_has_its_extra_fields(self) -> WriteBugReportArgs:
150
+ if self.kind == "conflict":
151
+ required = ("docs_excerpt", "affected_workflows", "suggested_resolution")
152
+ missing = [name for name in required if not getattr(self, name)]
153
+ if missing:
154
+ raise ValueError(f"a conflict report also needs: {', '.join(missing)}")
155
+ return self
156
+
157
+
158
+ class JsonPatchOp(BaseModel):
159
+ """One RFC 6902 operation."""
160
+
161
+ op: Literal["add", "remove", "replace", "move", "copy", "test"]
162
+ path: str = Field(pattern=r"^(/.*)?$")
163
+ value: Any = None
164
+ from_: str | None = Field(default=None, alias="from")
165
+
166
+
167
+ class ProposeSpecPatchArgs(BaseModel):
168
+ """Arguments for propose_spec_patch."""
169
+
170
+ test_id: str = Field(min_length=1)
171
+ patch: list[JsonPatchOp] = Field(min_length=1)
172
+
173
+
174
+ def _stub(milestone: str) -> Callable[[Any, ToolContext], str]:
175
+ def handler(args: Any, context: ToolContext) -> str:
176
+ raise ToolError(f"this tool is not implemented yet; it arrives in {milestone}")
177
+
178
+ return handler
179
+
180
+
181
+ def _workflows_dir(context: ToolContext) -> Path:
182
+ return context.workdir / "workflows"
183
+
184
+
185
+ def _load_workflow(context: ToolContext, workflow_id: str) -> Workflow:
186
+ path = _workflows_dir(context) / f"{workflow_id}.json"
187
+ if not path.is_file():
188
+ raise ToolError(f"no workflow named {workflow_id!r}")
189
+ try:
190
+ return Workflow.model_validate_json(path.read_text(encoding="utf-8"))
191
+ except (ValidationError, UnicodeDecodeError) as exc:
192
+ raise ToolError(f"{path.name} can't be read: {exc}") from exc
193
+
194
+
195
+ def read_spec(args: ReadSpecArgs, context: ToolContext) -> str:
196
+ """Read the API's operations, or one operation's parameters and response schemas."""
197
+ graph = context.graph
198
+ if graph is None:
199
+ raise ToolError("no spec is loaded")
200
+ if args.pointer is not None:
201
+ try:
202
+ return json.dumps(resolve_pointer(context.raw_spec or {}, args.pointer))
203
+ except JsonPointerException as exc:
204
+ raise ToolError(f"nothing at {args.pointer!r}: {exc}") from exc
205
+ if args.operation_id is None:
206
+ return json.dumps(
207
+ {
208
+ "operations": [
209
+ f"{op.operation_id}: {op.method} {op.path}" for op in graph.operations
210
+ ],
211
+ "links": [link.model_dump() for link in graph.links],
212
+ }
213
+ )
214
+ operation = graph.operation(args.operation_id)
215
+ if operation is None:
216
+ raise ToolError(f"no operation named {args.operation_id!r}")
217
+ return operation.model_dump_json(by_alias=True, exclude_none=True)
218
+
219
+
220
+ def read_docs(args: ReadDocsArgs, context: ToolContext) -> str:
221
+ """List the prose docs, or read one of them."""
222
+ root = context.docs_dir
223
+ if root is None:
224
+ return "No docs were provided: work from the spec."
225
+ root = root.resolve()
226
+ files = sorted(
227
+ str(path.relative_to(root))
228
+ for path in root.rglob("*")
229
+ if path.suffix.lower() in {".md", ".html", ".htm", ".txt"} and path.is_file()
230
+ )
231
+ if args.path is None:
232
+ return json.dumps(files)
233
+ if args.path not in files:
234
+ raise ToolError(f"no doc named {args.path!r}. Docs: {files}")
235
+ return (root / args.path).read_text(encoding="utf-8", errors="replace")
236
+
237
+
238
+ def read_workflows(args: ReadWorkflowsArgs, context: ToolContext) -> str:
239
+ """Read one workflow file, or all of them."""
240
+ if args.workflow_id is not None:
241
+ return _load_workflow(context, args.workflow_id).model_dump_json(exclude_none=True)
242
+ workflows = []
243
+ for path in sorted(_workflows_dir(context).glob("*.json")):
244
+ workflows.append(json.loads(path.read_text(encoding="utf-8")))
245
+ return json.dumps(workflows)
246
+
247
+
248
+ def _check_against_spec(workflow: Workflow, context: ToolContext) -> None:
249
+ graph = context.graph
250
+ if graph is None:
251
+ return
252
+ for step in [*workflow.steps, *workflow.cleanup]:
253
+ operation = graph.operation(step.op)
254
+ if operation is None:
255
+ raise ToolError(f"operation {step.op!r} isn't in the spec")
256
+ parameters = {p.name: p for p in operation.parameters}
257
+ unknown = sorted(set(step.params) - set(parameters))
258
+ if unknown:
259
+ raise ToolError(f"{step.op} has no parameters named {unknown}")
260
+ missing = sorted(
261
+ p.name
262
+ for p in parameters.values()
263
+ if p.location == "path" and p.name not in step.params
264
+ )
265
+ if missing:
266
+ raise ToolError(f"{step.op} needs path parameters {missing}")
267
+ _check_only_touches_own_data(workflow, graph)
268
+
269
+
270
+ def _is_generated(value: Any) -> bool:
271
+ """True when a value comes from a saved or unique placeholder, not a fixed literal."""
272
+ return isinstance(value, str) and "{" in value
273
+
274
+
275
+ def _check_only_touches_own_data(workflow: Workflow, graph: SpecGraph) -> None:
276
+ """Enforce NFR-3: tests must never change or delete data they didn't create."""
277
+ writes = {"PUT", "PATCH", "DELETE"}
278
+ for step in [*workflow.steps, *workflow.cleanup]:
279
+ operation = graph.operation(step.op)
280
+ if operation is None:
281
+ continue
282
+ if operation.method in writes:
283
+ path_names = {p.name for p in operation.parameters if p.location == "path"}
284
+ for name in sorted(path_names & set(step.params)):
285
+ if not _is_generated(step.params[name]):
286
+ raise ToolError(
287
+ f"{step.op} changes data, so {name} must come from a saved value or "
288
+ f"{{unique}}: a fixed value could hit data this test didn't create"
289
+ )
290
+ items = step.body if isinstance(step.body, list) else [step.body]
291
+ for item in items:
292
+ if isinstance(item, dict) and "id" in item and not _is_generated(item["id"]):
293
+ raise ToolError(
294
+ f"{step.op} sends a fixed id: use a saved value or {{unique_int}}, so it "
295
+ "can't overwrite data this test didn't create"
296
+ )
297
+ deletable = {op.resource for op in graph.operations if op.method == "DELETE"}
298
+ if not workflow.cleanup:
299
+ for step in workflow.steps:
300
+ operation = graph.operation(step.op)
301
+ if (
302
+ operation is not None
303
+ and operation.method == "POST"
304
+ and step.expect.status < 400
305
+ and operation.resource in deletable
306
+ ):
307
+ raise ToolError(
308
+ f"{step.op} creates data, so add a `cleanup` step that deletes it, even if a "
309
+ "step deletes it too: a failed test must not leave data behind"
310
+ )
311
+
312
+
313
+ def _expectation_changes(old: Workflow, new: Workflow) -> list[str]:
314
+ """List how a rewrite changes what the test checks, so weakening can't happen silently."""
315
+ changes = []
316
+ if len(new.steps) < len(old.steps):
317
+ changes.append(f"steps went from {len(old.steps)} to {len(new.steps)}")
318
+ for index, (before, after) in enumerate(zip(old.steps, new.steps, strict=False)):
319
+ if before.expect.status != after.expect.status:
320
+ changes.append(
321
+ f"steps[{index}] {before.op}: status {before.expect.status} "
322
+ f"became {after.expect.status}"
323
+ )
324
+ for key in sorted(set(before.expect.body) - set(after.expect.body)):
325
+ changes.append(f"steps[{index}] {before.op}: stopped checking {key}")
326
+ for key in sorted(set(before.expect.body) & set(after.expect.body)):
327
+ if before.expect.body[key] != after.expect.body[key]:
328
+ changes.append(f"steps[{index}] {before.op}: expected value of {key} changed")
329
+ return changes
330
+
331
+
332
+ def write_workflow(args: WriteWorkflowArgs, context: ToolContext) -> str:
333
+ """Create or replace workflows/<id>.json. Refuses locked files, and can't create them."""
334
+ # Only the pipeline marks failures as expected. A rewrite gets judged afresh on its next run.
335
+ workflow = args.workflow.model_copy(update={"known_issue": None})
336
+ if workflow.locked or workflow.source is Source.USER:
337
+ raise ToolError("agents can't create locked or user-sourced workflows")
338
+ _check_against_spec(workflow, context)
339
+ path = _workflows_dir(context) / f"{workflow.id}.json"
340
+ changes: list[str] = []
341
+ if path.exists():
342
+ try:
343
+ existing = Workflow.model_validate_json(path.read_text(encoding="utf-8"))
344
+ except (ValidationError, OSError, UnicodeDecodeError) as exc:
345
+ # It might be a user's locked file with a typo, so never overwrite what can't be read.
346
+ raise ToolError(f"{path.name} exists but can't be read; left unchanged") from exc
347
+ if existing.locked:
348
+ raise ToolError(f"{path.name} is locked by the user and can't be changed")
349
+ if context.new_workflows_only:
350
+ raise ToolError(
351
+ f"{path.name} already exists, and tests that have run stay as they are. "
352
+ "Write a new workflow with a new id."
353
+ )
354
+ changes = _expectation_changes(existing, workflow)
355
+ atomic_write_text(path, workflow.model_dump_json(indent=2, exclude_none=True) + "\n")
356
+ if changes:
357
+ log = context.workdir / ".workflowval" / "expectation_changes.jsonl"
358
+ log.parent.mkdir(parents=True, exist_ok=True)
359
+ with log.open("a", encoding="utf-8") as handle:
360
+ handle.write(json.dumps({"workflow_id": workflow.id, "changes": changes}) + "\n")
361
+ return f"wrote workflows/{workflow.id}.json"
362
+
363
+
364
+ def _review_path(context: ToolContext) -> Path:
365
+ return context.workdir / "review.json"
366
+
367
+
368
+ def _read_review_log(context: ToolContext) -> ReviewLog:
369
+ path = _review_path(context)
370
+ if not path.is_file():
371
+ return ReviewLog()
372
+ return ReviewLog.model_validate_json(path.read_text(encoding="utf-8"))
373
+
374
+
375
+ def _save_review_log(context: ToolContext, log: ReviewLog) -> None:
376
+ atomic_write_text(
377
+ _review_path(context), log.model_dump_json(indent=2, exclude_none=True) + "\n"
378
+ )
379
+
380
+
381
+ def write_review(args: WriteReviewArgs, context: ToolContext) -> str:
382
+ """Record review.json: workflows to fix and user journeys that are missing."""
383
+ log = _read_review_log(context)
384
+ known = {item.id for review in log.rounds for item in review.items()}
385
+ if repeated := sorted(known & {item.id for item in args.review.items()}):
386
+ raise ToolError(f"these item ids were used in an earlier round: {repeated}")
387
+ review = args.review.model_copy(
388
+ update={"round": len(log.rounds) + 1, "after_tests": context.new_workflows_only}
389
+ )
390
+ if context.new_workflows_only:
391
+ # A failure is a result, so a test that has run never changes because of one.
392
+ refusal = Answer(decision=Decision.REJECTED, reason="Tests that have run stay as they are.")
393
+ fixes = [fix.model_copy(update={"answer": refusal}) for fix in review.fixes]
394
+ review = review.model_copy(update={"fixes": fixes})
395
+ log.rounds.append(review)
396
+ _save_review_log(context, log)
397
+ return f"recorded review round {review.round}: {len(review.items())} items"
398
+
399
+
400
+ def answer_review(args: AnswerReviewArgs, context: ToolContext) -> str:
401
+ """Answer one review.json item: accept it naming a workflow, or reject it with a reason."""
402
+ log = _read_review_log(context)
403
+ item = log.item(args.item_id)
404
+ if item is None:
405
+ raise ToolError(f"no review item {args.item_id!r}. Open items: {log.unanswered()}")
406
+ workflow_id = args.answer.workflow_id
407
+ if workflow_id and not (_workflows_dir(context) / f"{workflow_id}.json").is_file():
408
+ raise ToolError(f"no workflow {workflow_id!r}: write it first, then accept this item")
409
+ item.answer = args.answer
410
+ _save_review_log(context, log)
411
+ return f"recorded the answer to {args.item_id}"
412
+
413
+
414
+ def write_test(args: WriteTestArgs, context: ToolContext) -> str:
415
+ """Replace tests/wf/test_<workflow_id>.py with fixed code. Assertions can't be weakened."""
416
+ workflow = _load_workflow(context, args.workflow_id)
417
+ path = context.workdir / test_path(args.workflow_id)
418
+ previous = path.read_text(encoding="utf-8") if path.is_file() else None
419
+ try:
420
+ check_test_source(args.source, workflow, previous)
421
+ body = args.source
422
+ if body.startswith("# workflow-sha256: "):
423
+ body = body.split("\n", 1)[1] if "\n" in body else ""
424
+ source = format_source(f"{hash_line(workflow)}\n{body}")
425
+ lint_source(source)
426
+ except SourceError as exc:
427
+ raise ToolError(str(exc)) from exc
428
+ atomic_write_text(path, source)
429
+ return f"wrote {test_path(args.workflow_id)}"
430
+
431
+
432
+ def read_tests(args: ReadTestsArgs, context: ToolContext) -> str:
433
+ """Read the generated test file for a workflow."""
434
+ path = context.workdir / test_path(args.workflow_id)
435
+ if not path.is_file():
436
+ raise ToolError(f"no test file for {args.workflow_id!r}")
437
+ return path.read_text(encoding="utf-8")
438
+
439
+
440
+ def run_tests(args: RunTestsArgs, context: ToolContext) -> str:
441
+ """Run the generated tests in a subprocess and record results."""
442
+ return _stub("a later release")(args, context)
443
+
444
+
445
+ def read_results(args: ReadResultsArgs, context: ToolContext) -> str:
446
+ """Read test results: status, failing assertion and the last 3 HTTP exchanges."""
447
+ path = context.workdir / ".workflowval" / "last_results.json"
448
+ if not path.is_file():
449
+ raise ToolError("no tests have run yet")
450
+ results = json.loads(path.read_text(encoding="utf-8"))
451
+ if args.test_id is None:
452
+ return json.dumps(results)
453
+ if args.test_id not in results:
454
+ raise ToolError(f"no result for {args.test_id!r}. Results: {sorted(results)}")
455
+ return json.dumps(results[args.test_id])
456
+
457
+
458
+ def http_probe(args: HttpProbeArgs, context: ToolContext) -> str:
459
+ """Send one request to an allowed host to investigate a failure."""
460
+ return _stub("a later release")(args, context)
461
+
462
+
463
+ def write_verdict(args: WriteVerdictArgs, context: ToolContext) -> str:
464
+ """Record the single label for a failed test, with its confidence and reason."""
465
+ if args.label is Label.UNRESOLVED:
466
+ raise ToolError("UNRESOLVED is set by the pipeline when fix attempts run out")
467
+ confidence = args.confidence
468
+ _, separator, workflow_id = args.test_id.rpartition("::test_")
469
+ if (
470
+ separator
471
+ and confidence is Confidence.HIGH
472
+ and _load_workflow(context, workflow_id).basis is Basis.ASSUMED
473
+ ):
474
+ confidence = Confidence.MED # Failures on assumed rules are capped at medium.
475
+ path = context.workdir / ".workflowval" / "verdicts.json"
476
+ verdicts = json.loads(path.read_text(encoding="utf-8")) if path.is_file() else {}
477
+ verdicts[args.test_id] = {
478
+ "label": args.label.value,
479
+ "confidence": confidence.value,
480
+ "reason": args.reason,
481
+ "fix_target": (args.fix_target or "workflow") if args.label is Label.TEST_BUG else None,
482
+ }
483
+ atomic_write_text(path, json.dumps(verdicts, indent=2) + "\n")
484
+ return f"recorded {args.label.value} for {args.test_id}"
485
+
486
+
487
+ def _evidence(context: ToolContext, target: str) -> dict[str, Any]:
488
+ path = context.workdir / ".workflowval" / "evidence.json"
489
+ evidence: dict[str, Any] = (
490
+ json.loads(path.read_text(encoding="utf-8")) if path.is_file() else {}
491
+ )
492
+ if target not in evidence:
493
+ raise ToolError(f"nothing was recorded for {target!r}. Recorded: {sorted(evidence)}")
494
+ found: dict[str, Any] = evidence[target]
495
+ return found
496
+
497
+
498
+ def _reports_dir(context: ToolContext) -> Path:
499
+ return context.reports_dir or context.workdir / "reports"
500
+
501
+
502
+ def _record_report(context: ToolContext, target: str, path: Path) -> str:
503
+ """Remember which file answers a test or finding, and return its path for the model."""
504
+ index = context.workdir / ".workflowval" / "reports.json"
505
+ written = json.loads(index.read_text(encoding="utf-8")) if index.is_file() else {}
506
+ written[target] = str(path)
507
+ atomic_write_text(index, json.dumps(written, indent=2) + "\n")
508
+ return str(path)
509
+
510
+
511
+ def _slug(text: str) -> str:
512
+ return re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-")[:60] or "report"
513
+
514
+
515
+ def write_bug_report(args: WriteBugReportArgs, context: ToolContext) -> str:
516
+ """Write a Markdown report for an API_BUG or a spec-docs CONFLICT. Code adds the curl steps."""
517
+ evidence = _evidence(context, args.test_id)
518
+ steps = "\n".join(curl_steps(evidence["calls"], context.auth))
519
+ tests = "\n".join(f"- `{test}`" for test in evidence.get("tests", [args.test_id]))
520
+ # The curl steps are masked already, and redacting them would hide the env vars they rely on.
521
+ before = [
522
+ f"# {args.title}",
523
+ f"**Kind:** {args.kind}",
524
+ f"## Tests that show it\n\n{tests}",
525
+ ]
526
+ after = [
527
+ f"## Expected\n\n{args.expected}",
528
+ f"## Actual\n\n{args.actual}",
529
+ f"## Spec reference\n\n{args.spec_reference}",
530
+ ]
531
+ if args.kind == "conflict":
532
+ workflows = "\n".join(f"- `{workflow}`" for workflow in args.affected_workflows)
533
+ after += [
534
+ f"## What the docs say\n\n{args.docs_excerpt}",
535
+ f"## Affected workflows\n\n{workflows}",
536
+ f"## Suggested resolution\n\n{args.suggested_resolution}",
537
+ ]
538
+ text = "\n\n".join(
539
+ [
540
+ redact_text("\n\n".join(before)),
541
+ f"## Steps to reproduce\n\n```\n{steps}\n```",
542
+ redact_text("\n\n".join(after)),
543
+ ]
544
+ )
545
+ path = _reports_dir(context) / "bugs" / f"{_slug(args.title)}.md"
546
+ path.parent.mkdir(parents=True, exist_ok=True)
547
+ atomic_write_text(path, text + "\n")
548
+ return f"wrote {_record_report(context, args.test_id, path)}"
549
+
550
+
551
+ def propose_spec_patch(args: ProposeSpecPatchArgs, context: ToolContext) -> str:
552
+ """Propose a minimal JSON Patch (RFC 6902) making the spec match observed behaviour."""
553
+ evidence = _evidence(context, args.test_id)
554
+ raw = context.raw_spec
555
+ if raw is None:
556
+ raise ToolError("no spec is loaded")
557
+ operations = [op.model_dump(by_alias=True, exclude_unset=True) for op in args.patch]
558
+ try:
559
+ patched = jsonpatch.apply_patch(raw, operations)
560
+ except (jsonpatch.JsonPatchException, JsonPointerException) as exc:
561
+ raise ToolError(f"the patch doesn't apply to the spec as written: {exc}") from exc
562
+ try:
563
+ checker = ResponseChecker(patched, validate_spec=True)
564
+ before_checker = ResponseChecker(raw)
565
+ except Exception as exc: # openapi-core raises many types for a broken spec.
566
+ raise ToolError(f"the patched spec isn't valid OpenAPI: {str(exc)[:300]}") from exc
567
+ calls = evidence["calls"]
568
+ wanted = evidence.get("keys")
569
+ before = {m.key for call in calls for m in before_checker.check_call(call)}
570
+ targets = set(wanted) if wanted else before
571
+ if not targets:
572
+ raise ToolError("the recorded response already fits the spec, so there's nothing to patch")
573
+ after = {m.key for call in calls for m in checker.check_call(call)}
574
+ if still := sorted(targets & after):
575
+ raise ToolError(f"the observed response still breaks the patched spec: {still}")
576
+ if added := sorted(after - before):
577
+ raise ToolError(f"the patch makes the observed response break the spec: {added}")
578
+ path = _reports_dir(context) / "patches" / f"{_slug(args.test_id)}.json"
579
+ path.parent.mkdir(parents=True, exist_ok=True)
580
+ atomic_write_text(path, json.dumps(operations, indent=2) + "\n")
581
+ return f"wrote {_record_report(context, args.test_id, path)}"
582
+
583
+
584
+ def _tool(fn: Callable[[Any, ToolContext], str], schema: type[BaseModel]) -> Tool:
585
+ if not fn.__doc__:
586
+ raise ValueError(f"tool {fn.__name__} needs a docstring: the model reads it")
587
+ return Tool(fn.__name__, fn.__doc__, schema, fn)
588
+
589
+
590
+ TOOLS: dict[str, Tool] = {
591
+ t.name: t
592
+ for t in (
593
+ _tool(read_spec, ReadSpecArgs),
594
+ _tool(read_docs, ReadDocsArgs),
595
+ _tool(read_workflows, ReadWorkflowsArgs),
596
+ _tool(write_workflow, WriteWorkflowArgs),
597
+ _tool(write_review, WriteReviewArgs),
598
+ _tool(answer_review, AnswerReviewArgs),
599
+ _tool(write_test, WriteTestArgs),
600
+ _tool(read_tests, ReadTestsArgs),
601
+ _tool(run_tests, RunTestsArgs),
602
+ _tool(read_results, ReadResultsArgs),
603
+ _tool(http_probe, HttpProbeArgs),
604
+ _tool(write_verdict, WriteVerdictArgs),
605
+ _tool(write_bug_report, WriteBugReportArgs),
606
+ _tool(propose_spec_patch, ProposeSpecPatchArgs),
607
+ )
608
+ }
609
+
610
+ ROLE_TOOLS: dict[str, frozenset[str]] = {
611
+ "planner": frozenset(
612
+ {
613
+ "read_spec",
614
+ "read_docs",
615
+ "read_workflows",
616
+ "write_workflow",
617
+ "answer_review",
618
+ "read_tests",
619
+ }
620
+ ),
621
+ "reviewer": frozenset(
622
+ {
623
+ "read_spec",
624
+ "read_docs",
625
+ "read_workflows",
626
+ "write_review",
627
+ "read_results",
628
+ "http_probe",
629
+ "write_verdict",
630
+ "write_bug_report",
631
+ "propose_spec_patch",
632
+ }
633
+ ),
634
+ "writer": frozenset(
635
+ {"read_spec", "read_workflows", "read_results", "write_test", "read_tests"}
636
+ ),
637
+ }