qaas-python 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- qaas/adapters/__init__.py +19 -0
- qaas/adapters/tracker.py +1350 -0
- qaas/adapters/vcs.py +494 -0
- qaas/cli.py +1564 -0
- qaas/conductor.py +527 -0
- qaas/config.py +407 -0
- qaas/defaults/config/agents/arbiter.yaml +19 -0
- qaas/defaults/config/agents/cartographer.yaml +20 -0
- qaas/defaults/config/agents/clerk.yaml +21 -0
- qaas/defaults/config/agents/conduit.yaml +19 -0
- qaas/defaults/config/agents/forge.yaml +22 -0
- qaas/defaults/config/agents/mender.yaml +56 -0
- qaas/defaults/config/agents/proof.yaml +21 -0
- qaas/defaults/config/agents/surface.yaml +16 -0
- qaas/defaults/config/system.yaml +69 -0
- qaas/discover.py +227 -0
- qaas/envelope.py +290 -0
- qaas/guardrails.py +431 -0
- qaas/mcp/__init__.py +0 -0
- qaas/mcp/context.py +70 -0
- qaas/mcp/contract_diff.py +937 -0
- qaas/mcp/defect_memory.py +495 -0
- qaas/mcp/env_control.py +905 -0
- qaas/mcp/envelope_server.py +463 -0
- qaas/mcp/test_runner.py +773 -0
- qaas/mcp/tracker.py +412 -0
- qaas/mcp/vcs.py +506 -0
- qaas/paths.py +317 -0
- qaas/plugin/.claude-plugin/plugin.json +9 -0
- qaas/plugin/skills/a11y-audit/SKILL.md +34 -0
- qaas/plugin/skills/adversarial-review/SKILL.md +120 -0
- qaas/plugin/skills/api-surface-extraction/SKILL.md +38 -0
- qaas/plugin/skills/authz-matrix-check/SKILL.md +46 -0
- qaas/plugin/skills/console-error-triage/SKILL.md +39 -0
- qaas/plugin/skills/contract-test-generation/SKILL.md +36 -0
- qaas/plugin/skills/dedupe-strategy/SKILL.md +39 -0
- qaas/plugin/skills/environment-pinning/SKILL.md +35 -0
- qaas/plugin/skills/error-taxonomy/SKILL.md +42 -0
- qaas/plugin/skills/exploratory-ui-walk/SKILL.md +46 -0
- qaas/plugin/skills/failing-test-authoring/SKILL.md +47 -0
- qaas/plugin/skills/flake-detection/SKILL.md +39 -0
- qaas/plugin/skills/form-state-probe/SKILL.md +36 -0
- qaas/plugin/skills/minimal-diff-discipline/SKILL.md +70 -0
- qaas/plugin/skills/openapi-diff/SKILL.md +45 -0
- qaas/plugin/skills/ownership-resolution/SKILL.md +31 -0
- qaas/plugin/skills/product-task-graph/SKILL.md +35 -0
- qaas/plugin/skills/regression-risk-scoring/SKILL.md +59 -0
- qaas/plugin/skills/regression-suite-selection/SKILL.md +36 -0
- qaas/plugin/skills/repo-cartography/SKILL.md +38 -0
- qaas/plugin/skills/repro-minimisation/SKILL.md +41 -0
- qaas/plugin/skills/rollback-plan-authoring/SKILL.md +81 -0
- qaas/plugin/skills/root-cause-vs-symptom/SKILL.md +67 -0
- qaas/plugin/skills/routing-rules/SKILL.md +34 -0
- qaas/plugin/skills/severity-rubric/SKILL.md +42 -0
- qaas/plugin/skills/test-first-fix/SKILL.md +66 -0
- qaas/plugin/skills/test-quality-audit/SKILL.md +58 -0
- qaas/plugin/skills/ticket-writer/SKILL.md +40 -0
- qaas/plugin/skills/verdict-reporting/SKILL.md +35 -0
- qaas/plugin/skills/verification-protocol/SKILL.md +39 -0
- qaas/prompts/ARBITER.md +53 -0
- qaas/prompts/CARTOGRAPHER.md +46 -0
- qaas/prompts/CLERK.md +45 -0
- qaas/prompts/CONDUIT.md +44 -0
- qaas/prompts/FORGE.md +43 -0
- qaas/prompts/MENDER.md +55 -0
- qaas/prompts/PROOF.md +41 -0
- qaas/prompts/SURFACE.md +46 -0
- qaas/prompts/_shared.md +45 -0
- qaas/registry.py +465 -0
- qaas/runner.py +192 -0
- qaas/scorecard.py +425 -0
- qaas/sdk_compat.py +52 -0
- qaas/store.py +290 -0
- qaas/target.py +261 -0
- qaas/tasks.py +361 -0
- qaas/trace.py +270 -0
- qaas_python-0.1.0.dist-info/METADATA +388 -0
- qaas_python-0.1.0.dist-info/RECORD +81 -0
- qaas_python-0.1.0.dist-info/WHEEL +4 -0
- qaas_python-0.1.0.dist-info/entry_points.txt +2 -0
- qaas_python-0.1.0.dist-info/licenses/LICENSE +21 -0
qaas/config.py
ADDED
|
@@ -0,0 +1,407 @@
|
|
|
1
|
+
"""Configuration: agents are data, not code.
|
|
2
|
+
|
|
3
|
+
An agent is a prompt file plus an entry in `config/agents/`. Adding one of the
|
|
4
|
+
remaining agents from the roster should never require touching the conductor,
|
|
5
|
+
the runner, or the guardrails — that is the property this module exists to keep.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any, Literal, Sequence
|
|
12
|
+
|
|
13
|
+
import os
|
|
14
|
+
|
|
15
|
+
import yaml
|
|
16
|
+
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
|
17
|
+
|
|
18
|
+
from qaas.target import TargetProfile, load_target
|
|
19
|
+
|
|
20
|
+
# §5.3 is explicit that no agent gets more than six MCP servers, because tool
|
|
21
|
+
# selection accuracy falls off past roughly 5-7. Enforced, not just documented.
|
|
22
|
+
MAX_MCP_SERVERS_PER_AGENT = 6
|
|
23
|
+
|
|
24
|
+
Layer = Literal["control", "discovery", "triage", "remediation", "reporting"]
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class Policy(BaseModel):
|
|
28
|
+
"""One agent's slice of the §8.1 write-permission matrix.
|
|
29
|
+
|
|
30
|
+
Default is read-only. Anything an agent may write, it says so here, and
|
|
31
|
+
guardrails.py enforces it against the actual tool call arguments.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
model_config = ConfigDict(extra="forbid")
|
|
35
|
+
|
|
36
|
+
write_paths: list[str] = Field(default_factory=list)
|
|
37
|
+
branch_patterns: list[str] = Field(default_factory=list)
|
|
38
|
+
may_open_pr: bool = False
|
|
39
|
+
may_create_tickets: bool = False
|
|
40
|
+
may_transition_tickets: bool = False
|
|
41
|
+
max_tickets_per_run: int = 0
|
|
42
|
+
max_diff_files: int | None = None
|
|
43
|
+
max_diff_lines: int | None = None
|
|
44
|
+
protected_paths: list[str] = Field(default_factory=list)
|
|
45
|
+
|
|
46
|
+
#: Path globs this agent may never modify, whatever else its policy allows.
|
|
47
|
+
#: §8.2 names the classes: migrations, auth, payment paths and infra config.
|
|
48
|
+
#: These are the changes whose blast radius a review cannot reliably bound,
|
|
49
|
+
#: so they stop at a human even when everything else in the envelope holds.
|
|
50
|
+
forbidden_paths: list[str] = Field(default_factory=list)
|
|
51
|
+
|
|
52
|
+
@property
|
|
53
|
+
def read_only(self) -> bool:
|
|
54
|
+
return not (
|
|
55
|
+
self.write_paths
|
|
56
|
+
or self.branch_patterns
|
|
57
|
+
or self.may_open_pr
|
|
58
|
+
or self.may_create_tickets
|
|
59
|
+
or self.may_transition_tickets
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class AgentSpec(BaseModel):
|
|
64
|
+
"""Everything needed to build one agent's ClaudeAgentOptions."""
|
|
65
|
+
|
|
66
|
+
model_config = ConfigDict(extra="forbid")
|
|
67
|
+
|
|
68
|
+
name: str
|
|
69
|
+
layer: Layer
|
|
70
|
+
role: str
|
|
71
|
+
prompt: str # path relative to src/qaas/prompts/
|
|
72
|
+
enabled: bool = True
|
|
73
|
+
|
|
74
|
+
model: str = "claude-opus-5"
|
|
75
|
+
effort: Literal["low", "medium", "high", "xhigh", "max"] = "high"
|
|
76
|
+
max_turns: int = 40
|
|
77
|
+
max_budget_usd: float = 2.0
|
|
78
|
+
|
|
79
|
+
mcp_servers: list[str] = Field(default_factory=list)
|
|
80
|
+
builtin_tools: list[str] = Field(default_factory=list)
|
|
81
|
+
policy: Policy = Field(default_factory=Policy)
|
|
82
|
+
|
|
83
|
+
# Procedure lives in skills, role and standards live in the prompt. A skill
|
|
84
|
+
# named here is preloaded; the agent can still reach others through Skill.
|
|
85
|
+
skills: list[str] = Field(default_factory=list)
|
|
86
|
+
|
|
87
|
+
# Tools this agent must have called before it is allowed to finish. The Stop
|
|
88
|
+
# hook enforces it. Without this an agent can produce a confident summary and
|
|
89
|
+
# no artifact, and the failure only surfaces afterwards in the conductor —
|
|
90
|
+
# too late for the agent to fix it.
|
|
91
|
+
must_call: list[str] = Field(default_factory=list)
|
|
92
|
+
|
|
93
|
+
@model_validator(mode="after")
|
|
94
|
+
def _tool_budget(self) -> "AgentSpec":
|
|
95
|
+
if len(self.mcp_servers) > MAX_MCP_SERVERS_PER_AGENT:
|
|
96
|
+
raise ValueError(
|
|
97
|
+
f"{self.name} declares {len(self.mcp_servers)} MCP servers; "
|
|
98
|
+
f"the cap is {MAX_MCP_SERVERS_PER_AGENT} (§5.3). "
|
|
99
|
+
"An agent needing more is a signal to split it."
|
|
100
|
+
)
|
|
101
|
+
if len(set(self.mcp_servers)) != len(self.mcp_servers):
|
|
102
|
+
raise ValueError(f"{self.name} lists a duplicate MCP server")
|
|
103
|
+
for tool in self.must_call:
|
|
104
|
+
server = tool.split("__")[1] if tool.startswith("mcp__") else None
|
|
105
|
+
if server and server not in self.mcp_servers:
|
|
106
|
+
raise ValueError(
|
|
107
|
+
f"{self.name} must_call names '{tool}' but is not connected to "
|
|
108
|
+
f"the '{server}' server; it could never satisfy that."
|
|
109
|
+
)
|
|
110
|
+
return self
|
|
111
|
+
|
|
112
|
+
def prompt_path(self, prompts_dir: Path) -> Path:
|
|
113
|
+
return prompts_dir / self.prompt
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
class StdioServerSpec(BaseModel):
|
|
117
|
+
"""A user-declared MCP server run as a subprocess.
|
|
118
|
+
|
|
119
|
+
Pure data: `command` and `args` are passed to the CLI, which spawns it. No
|
|
120
|
+
shell, ever -- `command` is a program and `args` is a list, so a string like
|
|
121
|
+
`"foo && rm -rf /"` is a program name that does not exist rather than two
|
|
122
|
+
commands.
|
|
123
|
+
"""
|
|
124
|
+
|
|
125
|
+
model_config = ConfigDict(extra="forbid")
|
|
126
|
+
|
|
127
|
+
type: Literal["stdio"] = "stdio"
|
|
128
|
+
command: str
|
|
129
|
+
args: list[str] = Field(default_factory=list)
|
|
130
|
+
env: dict[str, str] = Field(default_factory=dict)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
class UrlServerSpec(BaseModel):
|
|
134
|
+
"""A user-declared MCP server reached over HTTP or SSE."""
|
|
135
|
+
|
|
136
|
+
model_config = ConfigDict(extra="forbid")
|
|
137
|
+
|
|
138
|
+
type: Literal["http", "sse"]
|
|
139
|
+
url: str
|
|
140
|
+
headers: dict[str, str] = Field(default_factory=dict)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
#: What a user may declare. Deliberately no in-process Python type: that would
|
|
144
|
+
#: mean `importlib.import_module` on a name from a config file, executing
|
|
145
|
+
#: arbitrary module-level code inside the process holding this user's Anthropic
|
|
146
|
+
#: credentials, Jira token and GitHub auth. A subprocess is a subprocess; an
|
|
147
|
+
#: import is a foothold. If someone needs a Python server they can wrap it in a
|
|
148
|
+
#: stdio entry point and it costs them one line.
|
|
149
|
+
McpServerSpec = StdioServerSpec | UrlServerSpec
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
class Thresholds(BaseModel):
|
|
153
|
+
model_config = ConfigDict(extra="forbid")
|
|
154
|
+
|
|
155
|
+
min_confidence_to_file: float = 0.6
|
|
156
|
+
max_findings_per_agent_run: int = 25
|
|
157
|
+
max_tickets_per_run: int = 10
|
|
158
|
+
flake_runs: int = 5
|
|
159
|
+
max_mender_arbiter_round_trips: int = 2
|
|
160
|
+
max_proof_reopens: int = 1
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
class RunMode(BaseModel):
|
|
164
|
+
model_config = ConfigDict(extra="forbid")
|
|
165
|
+
|
|
166
|
+
trigger: str
|
|
167
|
+
agents: list[str]
|
|
168
|
+
max_budget_usd: float = 10.0
|
|
169
|
+
max_wall_clock_s: int = 3600
|
|
170
|
+
max_concurrency: int = 3
|
|
171
|
+
files_tickets: bool = True
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
class SystemConfig(BaseModel):
|
|
175
|
+
model_config = ConfigDict(extra="forbid")
|
|
176
|
+
|
|
177
|
+
project: str = "qaas"
|
|
178
|
+
|
|
179
|
+
#: Which target profile in config/targets/ this run is pointed at. The
|
|
180
|
+
#: profile is what makes the system portable: without it every prompt and
|
|
181
|
+
#: every environment call is welded to the application it was built beside.
|
|
182
|
+
#: The active target profile, or None when nothing is configured yet.
|
|
183
|
+
#: A fresh `pip install` is legitimately in that state; commands that
|
|
184
|
+
#: need a profile say so rather than crashing during config load.
|
|
185
|
+
target: str | None = None
|
|
186
|
+
|
|
187
|
+
#: Servers this project declares, on top of the built-in ones. Declaring a
|
|
188
|
+
#: server here grants nothing; an agent receives it only by naming it in its
|
|
189
|
+
#: own `mcp_servers:` list.
|
|
190
|
+
mcp_servers: dict[str, McpServerSpec] = Field(default_factory=dict)
|
|
191
|
+
|
|
192
|
+
#: Overridable with QAAS_TRACKER. Keep the committed value `local`.
|
|
193
|
+
tracker: Literal["local", "jira"] = "local"
|
|
194
|
+
vcs: Literal["local", "github"] = "local"
|
|
195
|
+
thresholds: Thresholds = Field(default_factory=Thresholds)
|
|
196
|
+
run_modes: dict[str, RunMode] = Field(default_factory=dict)
|
|
197
|
+
agents: dict[str, AgentSpec] = Field(default_factory=dict)
|
|
198
|
+
profile: TargetProfile | None = Field(default=None, exclude=True)
|
|
199
|
+
|
|
200
|
+
@model_validator(mode="after")
|
|
201
|
+
def _agents_name_real_servers(self) -> "SystemConfig":
|
|
202
|
+
"""Every server an agent names must resolve to something.
|
|
203
|
+
|
|
204
|
+
`AgentSpec` cannot check this -- it has no view of the rest of the
|
|
205
|
+
config -- so an unresolvable name used to surface as `UnknownServer`
|
|
206
|
+
part-way through a paid run. Here it is a load-time error, which is what
|
|
207
|
+
`qaas validate` is for.
|
|
208
|
+
|
|
209
|
+
Imported inside the function: `registry` imports `config`, so a
|
|
210
|
+
module-level import would be a cycle.
|
|
211
|
+
"""
|
|
212
|
+
from qaas.registry import SDK_SERVER_MODULES, STDIO_SERVERS
|
|
213
|
+
|
|
214
|
+
builtin = set(SDK_SERVER_MODULES) | set(STDIO_SERVERS)
|
|
215
|
+
known = builtin | set(self.mcp_servers)
|
|
216
|
+
for name, spec in sorted(self.agents.items()):
|
|
217
|
+
unknown = [s for s in spec.mcp_servers if s not in known]
|
|
218
|
+
if unknown:
|
|
219
|
+
raise ValueError(
|
|
220
|
+
f"{name} names MCP server(s) nothing provides: {', '.join(unknown)}. "
|
|
221
|
+
f"Built in: {', '.join(sorted(builtin))}. "
|
|
222
|
+
f"Declared in system.yaml: {', '.join(sorted(self.mcp_servers)) or 'none'}."
|
|
223
|
+
)
|
|
224
|
+
return self
|
|
225
|
+
|
|
226
|
+
@model_validator(mode="after")
|
|
227
|
+
def _modes_name_real_agents(self) -> "SystemConfig":
|
|
228
|
+
for mode_name, mode in self.run_modes.items():
|
|
229
|
+
unknown = [a for a in mode.agents if a not in self.agents]
|
|
230
|
+
if unknown:
|
|
231
|
+
raise ValueError(
|
|
232
|
+
f"run mode '{mode_name}' names unknown agents: {', '.join(unknown)}"
|
|
233
|
+
)
|
|
234
|
+
return self
|
|
235
|
+
|
|
236
|
+
def enabled_agents(self, mode: str) -> list[AgentSpec]:
|
|
237
|
+
"""Agents for a run mode, skipping any that are switched off."""
|
|
238
|
+
if mode not in self.run_modes:
|
|
239
|
+
raise KeyError(f"unknown run mode '{mode}'; have: {', '.join(sorted(self.run_modes))}")
|
|
240
|
+
return [self.agents[n] for n in self.run_modes[mode].agents if self.agents[n].enabled]
|
|
241
|
+
|
|
242
|
+
def target_root(self, base: Path | None = None) -> Path:
|
|
243
|
+
"""Where the application under test lives.
|
|
244
|
+
|
|
245
|
+
This used to be `Path.cwd() / config.target_app` -- one value serving as
|
|
246
|
+
both "where qaas lives" and "the application under test". That holds
|
|
247
|
+
only while the target is a subdirectory of the qaas checkout, which is
|
|
248
|
+
true of exactly one target: the bundled demo. `qaas run --repo <url>`
|
|
249
|
+
clones into `.qaas/targets/<slug>`, and every write-path allowlist,
|
|
250
|
+
every test cwd and the SDK subprocess cwd are anchored on this value --
|
|
251
|
+
so getting it from the profile is not tidying, it is the security
|
|
252
|
+
boundary being pointed at the right directory.
|
|
253
|
+
|
|
254
|
+
With no profile there is nothing to test; the base (the qaas project, or
|
|
255
|
+
the cwd) is returned so read-only tooling still has somewhere to stand.
|
|
256
|
+
"""
|
|
257
|
+
if self.profile is not None:
|
|
258
|
+
return self.profile.root_path(base)
|
|
259
|
+
from qaas.paths import project_root
|
|
260
|
+
|
|
261
|
+
return base if base is not None else project_root()
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
#: Environment overrides for the two swappable backends.
|
|
265
|
+
TRACKER_ENV = "QAAS_TRACKER"
|
|
266
|
+
VCS_ENV = "QAAS_VCS"
|
|
267
|
+
#: Which target profile to run against. Useful on its own (`QAAS_TARGET=staging
|
|
268
|
+
#: qaas run`), and it is how this repo's own test suite selects the bundled demo
|
|
269
|
+
#: without putting a demo name in the defaults that ship to everyone else.
|
|
270
|
+
TARGET_ENV = "QAAS_TARGET"
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def load_config(
|
|
274
|
+
config_dir: Path | str | None = None,
|
|
275
|
+
*,
|
|
276
|
+
search: Sequence[Path] | None = None,
|
|
277
|
+
target: str | None = None,
|
|
278
|
+
) -> SystemConfig:
|
|
279
|
+
"""Read system.yaml plus every agents/*.yaml, layered across search paths.
|
|
280
|
+
|
|
281
|
+
Passing `config_dir` positionally means "this directory and nothing else",
|
|
282
|
+
which is exactly the old behaviour and what every test does. Passing
|
|
283
|
+
`search` layers several directories: `system.yaml` is taken whole from the
|
|
284
|
+
first that has one, while `agents/*.yaml` and `targets/*.yaml` are unioned
|
|
285
|
+
by filename with earlier directories shadowing later ones -- so a user can
|
|
286
|
+
override one agent without forking all eight and freezing on today's roster.
|
|
287
|
+
|
|
288
|
+
With neither argument, the workspace resolver decides (an explicit
|
|
289
|
+
--config, then the project, then what shipped in the wheel).
|
|
290
|
+
|
|
291
|
+
`target` beats everything -- system.yaml, QAAS_TARGET, the single-profile
|
|
292
|
+
guess. It is what `qaas run --target X` and `qaas run --repo <url>` mean:
|
|
293
|
+
*this* application, whatever is configured. Without it, a stale `target:`
|
|
294
|
+
naming a profile that no longer exists killed the run inside config loading,
|
|
295
|
+
before the override the operator had just typed was ever consulted.
|
|
296
|
+
"""
|
|
297
|
+
if config_dir is not None:
|
|
298
|
+
dirs: list[Path] = [Path(config_dir)]
|
|
299
|
+
elif search is not None:
|
|
300
|
+
dirs = [Path(d) for d in search]
|
|
301
|
+
else:
|
|
302
|
+
from qaas.paths import Workspace
|
|
303
|
+
|
|
304
|
+
dirs = list(Workspace.resolve().config_dirs)
|
|
305
|
+
|
|
306
|
+
system_path = next((d / "system.yaml" for d in dirs if (d / "system.yaml").is_file()), None)
|
|
307
|
+
if system_path is None:
|
|
308
|
+
looked = ", ".join(str(d) for d in dirs) or "(nowhere -- no search path)"
|
|
309
|
+
raise FileNotFoundError(f"no system config at {dirs[0] / 'system.yaml'} (looked in: {looked})")
|
|
310
|
+
|
|
311
|
+
raw: dict[str, Any] = yaml.safe_load(system_path.read_text()) or {}
|
|
312
|
+
|
|
313
|
+
# `target_app:` used to name the application's directory relative to the
|
|
314
|
+
# process cwd. The target profile's `root` says the same thing and says it
|
|
315
|
+
# better, so the field is gone -- but `extra="forbid"` would turn an old
|
|
316
|
+
# system.yaml into a hard load failure, and someone else's committed config
|
|
317
|
+
# is not ours to break. Dropped silently: there is nothing for the reader to
|
|
318
|
+
# do about it, and the profile already carries the answer.
|
|
319
|
+
raw.pop("target_app", None)
|
|
320
|
+
|
|
321
|
+
# Backend overrides from the environment, so pointing a run at a real
|
|
322
|
+
# tracker or forge is not a committed file change.
|
|
323
|
+
#
|
|
324
|
+
# `tracker: local` is the committed default and must stay that way. When
|
|
325
|
+
# `jira` was committed instead, 18 tests failed and 14 errored: the agent
|
|
326
|
+
# fixtures build a real JiraTracker, which demands credentials CI does not
|
|
327
|
+
# have. The house rule is that the default `pytest` run is offline and free,
|
|
328
|
+
# and a committed backend switch silently breaks it -- so the switch belongs
|
|
329
|
+
# in the environment of the person who wants it, not in the repo.
|
|
330
|
+
for key, var in (("tracker", TRACKER_ENV), ("vcs", VCS_ENV), ("target", TARGET_ENV)):
|
|
331
|
+
raw_value = (os.environ.get(var) or "").strip()
|
|
332
|
+
if raw_value:
|
|
333
|
+
# Backends are lowercase literals; a target is a profile name.
|
|
334
|
+
raw[key] = raw_value if key == "target" else raw_value.lower()
|
|
335
|
+
|
|
336
|
+
# An explicit argument outranks both the file and the environment.
|
|
337
|
+
if target:
|
|
338
|
+
raw["target"] = target
|
|
339
|
+
|
|
340
|
+
# Agents layer by filename. Walking the search paths in reverse means the
|
|
341
|
+
# highest-precedence directory writes last and therefore wins.
|
|
342
|
+
by_stem: dict[str, Path] = {}
|
|
343
|
+
for d in reversed(dirs):
|
|
344
|
+
for path in sorted((d / "agents").glob("*.yaml")):
|
|
345
|
+
by_stem[path.stem] = path
|
|
346
|
+
|
|
347
|
+
agents: dict[str, Any] = {}
|
|
348
|
+
for path in by_stem.values():
|
|
349
|
+
spec = yaml.safe_load(path.read_text()) or {}
|
|
350
|
+
name = spec.get("name") or path.stem.upper()
|
|
351
|
+
spec["name"] = name
|
|
352
|
+
if name in agents:
|
|
353
|
+
raise ValueError(f"duplicate agent definition for {name} at {path}")
|
|
354
|
+
agents[name] = spec
|
|
355
|
+
|
|
356
|
+
raw["agents"] = agents
|
|
357
|
+
|
|
358
|
+
config = SystemConfig.model_validate(raw)
|
|
359
|
+
|
|
360
|
+
# Resolve the target profile if one is named and findable.
|
|
361
|
+
#
|
|
362
|
+
# A named-but-missing profile stays fatal -- running the wrong application
|
|
363
|
+
# is worse than not running. But `target: null` is a legitimate state now:
|
|
364
|
+
# `pip install qaas-python` gives you a working CLI that is not yet pointed
|
|
365
|
+
# at anything, and the commands that need a profile say "run qaas init"
|
|
366
|
+
# rather than dying inside config loading.
|
|
367
|
+
# Profiles layer by filename, exactly as agents and skills do. This used to
|
|
368
|
+
# take the *first* config layer that had a `targets/` at all -- and the day
|
|
369
|
+
# `qaas run --repo` started writing a generated profile into the writable
|
|
370
|
+
# layer (`.qaas/config/targets/`), that layer became "the" targets directory
|
|
371
|
+
# and every profile in `<project>/config/targets/` vanished: `qaas doctor
|
|
372
|
+
# --target corvid` reported the demo profile did not exist.
|
|
373
|
+
profiles = target_files(dirs)
|
|
374
|
+
chosen = config.target
|
|
375
|
+
|
|
376
|
+
# No target named, but exactly one profile on disk: use it. Choosing between
|
|
377
|
+
# two would be guessing, and running the wrong application is worse than not
|
|
378
|
+
# running -- but with one candidate there is nothing to guess at, and making
|
|
379
|
+
# the user restate it is ceremony. This is also what keeps this repository
|
|
380
|
+
# working: its `system.yaml` ships in the package and names no target,
|
|
381
|
+
# because a demo name has no business in the defaults everyone installs.
|
|
382
|
+
if not chosen and len(profiles) == 1:
|
|
383
|
+
chosen = next(iter(profiles))
|
|
384
|
+
|
|
385
|
+
if chosen and profiles:
|
|
386
|
+
if chosen not in profiles:
|
|
387
|
+
# Named but absent stays fatal: running the wrong application is
|
|
388
|
+
# worse than not running. Listed from the merged view, so the
|
|
389
|
+
# suggestion names every profile the user actually has.
|
|
390
|
+
raise FileNotFoundError(
|
|
391
|
+
f"no target profile '{chosen}'. Available: {', '.join(sorted(profiles))}. "
|
|
392
|
+
"Create one with `qaas init <path-to-repo>`."
|
|
393
|
+
)
|
|
394
|
+
profile = load_target(chosen, profiles[chosen].parent)
|
|
395
|
+
config = config.model_copy(update={"target": chosen, "profile": profile})
|
|
396
|
+
return config
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
def target_files(dirs: Sequence[Path]) -> dict[str, Path]:
|
|
400
|
+
"""Every target profile visible across the config layers, nearest wins."""
|
|
401
|
+
found: dict[str, Path] = {}
|
|
402
|
+
for d in reversed(list(dirs)):
|
|
403
|
+
base = Path(d) / "targets"
|
|
404
|
+
if base.is_dir():
|
|
405
|
+
for path in sorted(base.glob("*.yaml")):
|
|
406
|
+
found[path.stem] = path
|
|
407
|
+
return found
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
name: ARBITER
|
|
2
|
+
layer: remediation
|
|
3
|
+
role: >
|
|
4
|
+
Review and risk gate. Reads MENDER's diff as an adversarial reviewer and judges
|
|
5
|
+
correctness, scope creep, hidden regressions and risk. Separate from MENDER for
|
|
6
|
+
the same reason discovery is separate from triage: a model reviewing its own
|
|
7
|
+
diff in the same context reliably rationalises it.
|
|
8
|
+
prompt: ARBITER.md
|
|
9
|
+
model: claude-opus-5
|
|
10
|
+
effort: high
|
|
11
|
+
max_turns: 50
|
|
12
|
+
max_budget_usd: 3.0
|
|
13
|
+
mcp_servers: [envelope, vcs, contract_diff, test_runner]
|
|
14
|
+
builtin_tools: [Read, Grep, Glob]
|
|
15
|
+
skills: [adversarial-review, root-cause-vs-symptom, regression-risk-scoring, test-quality-audit]
|
|
16
|
+
|
|
17
|
+
policy: {} # no write access to code, ever — the whole point of the separation
|
|
18
|
+
|
|
19
|
+
must_call: [mcp__envelope__record_review]
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
name: CARTOGRAPHER
|
|
2
|
+
layer: control
|
|
3
|
+
role: >
|
|
4
|
+
Builds and maintains the shared system map every other agent reads: services,
|
|
5
|
+
routes, API surface, schema, dependency graph, ownership.
|
|
6
|
+
prompt: CARTOGRAPHER.md
|
|
7
|
+
model: claude-sonnet-5 # extraction, not judgment
|
|
8
|
+
effort: medium
|
|
9
|
+
max_turns: 60
|
|
10
|
+
max_budget_usd: 2.0
|
|
11
|
+
mcp_servers: [envelope]
|
|
12
|
+
builtin_tools: [Read, Grep, Glob]
|
|
13
|
+
policy: {} # read-only
|
|
14
|
+
|
|
15
|
+
# Procedure lives in skills; the prompt carries role and standards.
|
|
16
|
+
skills: [repo-cartography, api-surface-extraction, ownership-resolution, product-task-graph]
|
|
17
|
+
|
|
18
|
+
# The map is the deliverable. An agent that finishes without publishing one has
|
|
19
|
+
# not done the job, and the Stop hook says so while it can still act on that.
|
|
20
|
+
must_call: [mcp__envelope__put_system_map]
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
name: CLERK
|
|
2
|
+
layer: triage
|
|
3
|
+
role: >
|
|
4
|
+
Triage and ticket scribe. The only agent with tracker write access. Dedupes,
|
|
5
|
+
scores severity against the rubric, resolves ownership from the system map,
|
|
6
|
+
routes by class, and files.
|
|
7
|
+
prompt: CLERK.md
|
|
8
|
+
model: claude-sonnet-5 # composition against a fixed rubric and house format
|
|
9
|
+
effort: high
|
|
10
|
+
max_turns: 50
|
|
11
|
+
max_budget_usd: 2.0
|
|
12
|
+
mcp_servers: [envelope, tracker, defect_memory]
|
|
13
|
+
builtin_tools: [Read]
|
|
14
|
+
policy:
|
|
15
|
+
may_create_tickets: true
|
|
16
|
+
max_tickets_per_run: 10 # §4.12: over cap, pause and escalate
|
|
17
|
+
|
|
18
|
+
skills: [severity-rubric, dedupe-strategy, ticket-writer, routing-rules, ownership-resolution]
|
|
19
|
+
|
|
20
|
+
# Filing nothing is allowed; filing without checking for duplicates is not.
|
|
21
|
+
must_call: [mcp__defect_memory__search_similar]
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
name: CONDUIT
|
|
2
|
+
layer: discovery
|
|
3
|
+
role: >
|
|
4
|
+
Backend, API and contract analyst. Finds spec drift, breaking changes, missing
|
|
5
|
+
authorization, error-taxonomy inconsistency, unbounded results, validation gaps.
|
|
6
|
+
Ships a failing contract test as evidence, never a bare opinion.
|
|
7
|
+
prompt: CONDUIT.md
|
|
8
|
+
model: claude-opus-5
|
|
9
|
+
effort: high
|
|
10
|
+
max_turns: 60
|
|
11
|
+
max_budget_usd: 3.0
|
|
12
|
+
mcp_servers: [envelope, contract_diff, env_control, defect_memory]
|
|
13
|
+
builtin_tools: [Read, Grep, Glob]
|
|
14
|
+
policy: {}
|
|
15
|
+
|
|
16
|
+
skills: [openapi-diff, authz-matrix-check, error-taxonomy, contract-test-generation, severity-rubric]
|
|
17
|
+
|
|
18
|
+
# No must_call: finding nothing is a valid and useful outcome for a discovery
|
|
19
|
+
# agent, and requiring an emission would manufacture findings to satisfy it.
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
name: FORGE
|
|
2
|
+
layer: triage
|
|
3
|
+
role: >
|
|
4
|
+
Reproduction engineer. Turns a draft finding into a deterministic minimal
|
|
5
|
+
reproduction plus a failing test, measures flake rate, and demotes what it
|
|
6
|
+
cannot reproduce. The noise filter the whole system depends on.
|
|
7
|
+
prompt: FORGE.md
|
|
8
|
+
model: claude-opus-5
|
|
9
|
+
effort: high
|
|
10
|
+
max_turns: 80
|
|
11
|
+
max_budget_usd: 4.0
|
|
12
|
+
mcp_servers: [envelope, test_runner, env_control, vcs]
|
|
13
|
+
builtin_tools: [Read, Grep, Glob, Write, Edit, Bash]
|
|
14
|
+
policy:
|
|
15
|
+
write_paths: ["qa/repro"] # sandboxed workspace only (§8.1)
|
|
16
|
+
branch_patterns: ["qa/repro/*"] # never main, never force-push
|
|
17
|
+
|
|
18
|
+
skills: [repro-minimisation, failing-test-authoring, flake-detection, environment-pinning]
|
|
19
|
+
|
|
20
|
+
# FORGE is handed exactly one finding and owes exactly one verdict on it.
|
|
21
|
+
# Silence here would let an unreproduced finding drift toward a ticket.
|
|
22
|
+
must_call: [mcp__envelope__record_reproduction]
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
name: MENDER
|
|
2
|
+
layer: remediation
|
|
3
|
+
role: >
|
|
4
|
+
Remediation engineer. Picks up an agent-ready ticket, writes the minimal change
|
|
5
|
+
that makes the failing test pass, and opens a draft pull request. Deliberately
|
|
6
|
+
one agent with many skills rather than five domain-specific fixers: fixing is
|
|
7
|
+
one activity, and the domain knowledge belongs in loadable skills, not in five
|
|
8
|
+
nearly identical prompts.
|
|
9
|
+
prompt: MENDER.md
|
|
10
|
+
model: claude-opus-5
|
|
11
|
+
effort: high
|
|
12
|
+
max_turns: 80
|
|
13
|
+
max_budget_usd: 5.0
|
|
14
|
+
mcp_servers: [envelope, test_runner, env_control, vcs, tracker, contract_diff]
|
|
15
|
+
builtin_tools: [Read, Grep, Glob, Write, Edit, Bash]
|
|
16
|
+
skills: [test-first-fix, minimal-diff-discipline, root-cause-vs-symptom, rollback-plan-authoring]
|
|
17
|
+
|
|
18
|
+
policy:
|
|
19
|
+
# Product code, on its own branches. Relative to the *target's* root, not to
|
|
20
|
+
# the qaas project: these used to read `target-app/api/app`, which only ever
|
|
21
|
+
# resolved because the demo happened to live inside this checkout. Still set
|
|
22
|
+
# per target -- a different application keeps its source somewhere else.
|
|
23
|
+
write_paths: [api/app, web/src, qa/repro]
|
|
24
|
+
branch_patterns: ["fix/*"]
|
|
25
|
+
may_open_pr: true
|
|
26
|
+
may_transition_tickets: true
|
|
27
|
+
|
|
28
|
+
# The §8.2 autonomy envelope, enforced in guardrails rather than asked for in
|
|
29
|
+
# the prompt. Start tight; widen from measured ARBITER approval and PROOF
|
|
30
|
+
# verification rates, not from optimism.
|
|
31
|
+
max_diff_files: 5
|
|
32
|
+
max_diff_lines: 150
|
|
33
|
+
|
|
34
|
+
# §8.2: these classes stop at a human however small the change looks. Their
|
|
35
|
+
# blast radius is not something a review can reliably bound.
|
|
36
|
+
#
|
|
37
|
+
# Matched against a path relative to the *target's* root, so the directory
|
|
38
|
+
# patterns are `*x/*` and not `*/x/*`: with a leading slash required,
|
|
39
|
+
# `.github/workflows/ci.yml` -- a repository's CI at its own root, which is
|
|
40
|
+
# where almost every repository keeps it -- matched nothing. That hole was
|
|
41
|
+
# invisible while paths arrived prefixed with `target-app/`.
|
|
42
|
+
forbidden_paths:
|
|
43
|
+
- "*migrations/*"
|
|
44
|
+
- "*migration*"
|
|
45
|
+
- "*auth.py"
|
|
46
|
+
- "*auth*"
|
|
47
|
+
- "*payment*"
|
|
48
|
+
- "*billing*"
|
|
49
|
+
- "*secret*"
|
|
50
|
+
- "*.tf"
|
|
51
|
+
- "*infra/*"
|
|
52
|
+
- "*docker-compose*"
|
|
53
|
+
- "Dockerfile*"
|
|
54
|
+
- "*.github/*"
|
|
55
|
+
|
|
56
|
+
must_call: [mcp__vcs__open_pr]
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
name: PROOF
|
|
2
|
+
layer: remediation
|
|
3
|
+
role: >
|
|
4
|
+
Verification and regression gate, and the closing authority. Re-runs the
|
|
5
|
+
original failing test against the patched build, runs the regression suite for
|
|
6
|
+
affected areas, and returns VERIFIED, NOT_FIXED or REGRESSED.
|
|
7
|
+
prompt: PROOF.md
|
|
8
|
+
model: claude-opus-5
|
|
9
|
+
effort: high
|
|
10
|
+
max_turns: 60
|
|
11
|
+
max_budget_usd: 4.0
|
|
12
|
+
mcp_servers: [envelope, test_runner, env_control, tracker, vcs]
|
|
13
|
+
builtin_tools: [Read, Grep, Glob, Bash]
|
|
14
|
+
policy:
|
|
15
|
+
may_transition_tickets: true # transition only, never create (§8.1)
|
|
16
|
+
|
|
17
|
+
skills: [verification-protocol, regression-suite-selection, verdict-reporting, severity-rubric]
|
|
18
|
+
|
|
19
|
+
# A verification that ends without transitioning the ticket has left the ticket
|
|
20
|
+
# in review forever, which is the one outcome worse than a wrong verdict.
|
|
21
|
+
must_call: [mcp__envelope__record_verdict, mcp__tracker__transition]
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
name: SURFACE
|
|
2
|
+
layer: discovery
|
|
3
|
+
role: >
|
|
4
|
+
Frontend and UI explorer. Walks the product as a user does: broken flows,
|
|
5
|
+
console errors, accessibility failures, missing loading/empty/error states,
|
|
6
|
+
form validation gaps, state desync.
|
|
7
|
+
prompt: SURFACE.md
|
|
8
|
+
model: claude-opus-5
|
|
9
|
+
effort: high
|
|
10
|
+
max_turns: 80
|
|
11
|
+
max_budget_usd: 4.0
|
|
12
|
+
mcp_servers: [envelope, env_control, playwright]
|
|
13
|
+
builtin_tools: [Read, Grep, Glob]
|
|
14
|
+
policy: {}
|
|
15
|
+
|
|
16
|
+
skills: [exploratory-ui-walk, a11y-audit, console-error-triage, form-state-probe, product-task-graph, severity-rubric]
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
# System configuration. Run modes follow architecture §9; the Phase 1 roster is
|
|
2
|
+
# the six agents from §11, so modes name only those until Phase 2 lands.
|
|
3
|
+
project: qaas
|
|
4
|
+
# Which target profile to run against, from <config>/targets/<name>.yaml.
|
|
5
|
+
# Unset means "not pointed at anything yet", which is exactly what a fresh
|
|
6
|
+
# `pip install` is. `qaas init` writes this for you; QAAS_TARGET overrides it.
|
|
7
|
+
target:
|
|
8
|
+
|
|
9
|
+
tracker: local # local | jira -- swap to file against real Jira
|
|
10
|
+
vcs: local # local | github
|
|
11
|
+
|
|
12
|
+
thresholds:
|
|
13
|
+
min_confidence_to_file: 0.6 # §7 confidence gate
|
|
14
|
+
max_findings_per_agent_run: 25 # §8.3 loop breaker: pause and escalate, don't file
|
|
15
|
+
max_tickets_per_run: 10 # §4.12 rate limit
|
|
16
|
+
flake_runs: 5 # FORGE runs a repro N times to measure flake
|
|
17
|
+
max_mender_arbiter_round_trips: 2
|
|
18
|
+
max_proof_reopens: 1
|
|
19
|
+
|
|
20
|
+
run_modes:
|
|
21
|
+
pr-check:
|
|
22
|
+
trigger: pull_request
|
|
23
|
+
agents: [CARTOGRAPHER, CONDUIT, SURFACE, FORGE, CLERK]
|
|
24
|
+
# Was 6.0, which this roster could not complete. Measured on a real run:
|
|
25
|
+
# CARTOGRAPHER $0.69 + CONDUIT $2.25 + SURFACE $3.16 = $6.09, so the governor
|
|
26
|
+
# stopped the run before FORGE or CLERK ever dispatched. The mode meant to
|
|
27
|
+
# run on every pull request could not file a ticket, and it failed silently:
|
|
28
|
+
# agents ran, findings landed in the ledger, nothing errored. `qaas validate`
|
|
29
|
+
# now refuses a mode whose agents cannot fit inside its cap.
|
|
30
|
+
max_budget_usd: 16.0
|
|
31
|
+
max_wall_clock_s: 900
|
|
32
|
+
max_concurrency: 2
|
|
33
|
+
|
|
34
|
+
nightly:
|
|
35
|
+
trigger: cron
|
|
36
|
+
agents: [CARTOGRAPHER, CONDUIT, SURFACE, FORGE, CLERK]
|
|
37
|
+
# FORGE runs once per finding, so the deep sweep's budget scales with how
|
|
38
|
+
# much discovery found, not with the number of agents. Measured: discovery
|
|
39
|
+
# ~$6, then roughly $1-2 per finding reproduced.
|
|
40
|
+
max_budget_usd: 40.0
|
|
41
|
+
max_wall_clock_s: 7200
|
|
42
|
+
max_concurrency: 3
|
|
43
|
+
|
|
44
|
+
incident:
|
|
45
|
+
trigger: alert
|
|
46
|
+
agents: [CONDUIT]
|
|
47
|
+
max_budget_usd: 4.0
|
|
48
|
+
max_wall_clock_s: 600
|
|
49
|
+
max_concurrency: 2
|
|
50
|
+
files_tickets: false # §9: diagnostic only, read-only, no filing
|
|
51
|
+
|
|
52
|
+
# §9: the remediation loop. PROOF verifies first — a ticket whose defect no
|
|
53
|
+
# longer reproduces needs no fix, and finding that out costs one cheap run
|
|
54
|
+
# instead of a whole MENDER/ARBITER cycle.
|
|
55
|
+
fix-cycle:
|
|
56
|
+
trigger: agent_ready_ticket
|
|
57
|
+
agents: [PROOF, MENDER, ARBITER]
|
|
58
|
+
max_budget_usd: 20.0
|
|
59
|
+
max_wall_clock_s: 3600
|
|
60
|
+
max_concurrency: 1
|
|
61
|
+
|
|
62
|
+
# Everything, end to end: discover, triage, file, fix, review, verify. The most
|
|
63
|
+
# expensive mode in the system and the only one that closes the loop.
|
|
64
|
+
full-loop:
|
|
65
|
+
trigger: on_demand
|
|
66
|
+
agents: [CARTOGRAPHER, CONDUIT, SURFACE, FORGE, CLERK, MENDER, ARBITER, PROOF]
|
|
67
|
+
max_budget_usd: 60.0
|
|
68
|
+
max_wall_clock_s: 10800
|
|
69
|
+
max_concurrency: 3
|