qaas-python 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- qaas/adapters/__init__.py +19 -0
- qaas/adapters/tracker.py +1350 -0
- qaas/adapters/vcs.py +494 -0
- qaas/cli.py +1564 -0
- qaas/conductor.py +527 -0
- qaas/config.py +407 -0
- qaas/defaults/config/agents/arbiter.yaml +19 -0
- qaas/defaults/config/agents/cartographer.yaml +20 -0
- qaas/defaults/config/agents/clerk.yaml +21 -0
- qaas/defaults/config/agents/conduit.yaml +19 -0
- qaas/defaults/config/agents/forge.yaml +22 -0
- qaas/defaults/config/agents/mender.yaml +56 -0
- qaas/defaults/config/agents/proof.yaml +21 -0
- qaas/defaults/config/agents/surface.yaml +16 -0
- qaas/defaults/config/system.yaml +69 -0
- qaas/discover.py +227 -0
- qaas/envelope.py +290 -0
- qaas/guardrails.py +431 -0
- qaas/mcp/__init__.py +0 -0
- qaas/mcp/context.py +70 -0
- qaas/mcp/contract_diff.py +937 -0
- qaas/mcp/defect_memory.py +495 -0
- qaas/mcp/env_control.py +905 -0
- qaas/mcp/envelope_server.py +463 -0
- qaas/mcp/test_runner.py +773 -0
- qaas/mcp/tracker.py +412 -0
- qaas/mcp/vcs.py +506 -0
- qaas/paths.py +317 -0
- qaas/plugin/.claude-plugin/plugin.json +9 -0
- qaas/plugin/skills/a11y-audit/SKILL.md +34 -0
- qaas/plugin/skills/adversarial-review/SKILL.md +120 -0
- qaas/plugin/skills/api-surface-extraction/SKILL.md +38 -0
- qaas/plugin/skills/authz-matrix-check/SKILL.md +46 -0
- qaas/plugin/skills/console-error-triage/SKILL.md +39 -0
- qaas/plugin/skills/contract-test-generation/SKILL.md +36 -0
- qaas/plugin/skills/dedupe-strategy/SKILL.md +39 -0
- qaas/plugin/skills/environment-pinning/SKILL.md +35 -0
- qaas/plugin/skills/error-taxonomy/SKILL.md +42 -0
- qaas/plugin/skills/exploratory-ui-walk/SKILL.md +46 -0
- qaas/plugin/skills/failing-test-authoring/SKILL.md +47 -0
- qaas/plugin/skills/flake-detection/SKILL.md +39 -0
- qaas/plugin/skills/form-state-probe/SKILL.md +36 -0
- qaas/plugin/skills/minimal-diff-discipline/SKILL.md +70 -0
- qaas/plugin/skills/openapi-diff/SKILL.md +45 -0
- qaas/plugin/skills/ownership-resolution/SKILL.md +31 -0
- qaas/plugin/skills/product-task-graph/SKILL.md +35 -0
- qaas/plugin/skills/regression-risk-scoring/SKILL.md +59 -0
- qaas/plugin/skills/regression-suite-selection/SKILL.md +36 -0
- qaas/plugin/skills/repo-cartography/SKILL.md +38 -0
- qaas/plugin/skills/repro-minimisation/SKILL.md +41 -0
- qaas/plugin/skills/rollback-plan-authoring/SKILL.md +81 -0
- qaas/plugin/skills/root-cause-vs-symptom/SKILL.md +67 -0
- qaas/plugin/skills/routing-rules/SKILL.md +34 -0
- qaas/plugin/skills/severity-rubric/SKILL.md +42 -0
- qaas/plugin/skills/test-first-fix/SKILL.md +66 -0
- qaas/plugin/skills/test-quality-audit/SKILL.md +58 -0
- qaas/plugin/skills/ticket-writer/SKILL.md +40 -0
- qaas/plugin/skills/verdict-reporting/SKILL.md +35 -0
- qaas/plugin/skills/verification-protocol/SKILL.md +39 -0
- qaas/prompts/ARBITER.md +53 -0
- qaas/prompts/CARTOGRAPHER.md +46 -0
- qaas/prompts/CLERK.md +45 -0
- qaas/prompts/CONDUIT.md +44 -0
- qaas/prompts/FORGE.md +43 -0
- qaas/prompts/MENDER.md +55 -0
- qaas/prompts/PROOF.md +41 -0
- qaas/prompts/SURFACE.md +46 -0
- qaas/prompts/_shared.md +45 -0
- qaas/registry.py +465 -0
- qaas/runner.py +192 -0
- qaas/scorecard.py +425 -0
- qaas/sdk_compat.py +52 -0
- qaas/store.py +290 -0
- qaas/target.py +261 -0
- qaas/tasks.py +361 -0
- qaas/trace.py +270 -0
- qaas_python-0.1.0.dist-info/METADATA +388 -0
- qaas_python-0.1.0.dist-info/RECORD +81 -0
- qaas_python-0.1.0.dist-info/WHEEL +4 -0
- qaas_python-0.1.0.dist-info/entry_points.txt +2 -0
- qaas_python-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,388 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: qaas-python
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A multi-agent QA system: finds real defects, reproduces them, files tickets, fixes them, and proves the fix
|
|
5
|
+
Project-URL: Homepage, https://github.com/allaabdella2-us/qa-multi-agent-system
|
|
6
|
+
Project-URL: Repository, https://github.com/allaabdella2-us/qa-multi-agent-system
|
|
7
|
+
Project-URL: Issues, https://github.com/allaabdella2-us/qa-multi-agent-system/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/allaabdella2-us/qa-multi-agent-system/blob/main/CHANGELOG.md
|
|
9
|
+
Author: Alla Abdella
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: agents,bug-finding,claude,llm,multi-agent,qa,testing
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
19
|
+
Classifier: Topic :: Software Development :: Testing
|
|
20
|
+
Requires-Python: >=3.12
|
|
21
|
+
Requires-Dist: claude-agent-sdk>=0.2.127
|
|
22
|
+
Requires-Dist: pydantic>=2.9
|
|
23
|
+
Requires-Dist: pyyaml>=6.0
|
|
24
|
+
Requires-Dist: rich>=13.9
|
|
25
|
+
Requires-Dist: typer>=0.15
|
|
26
|
+
Provides-Extra: dev
|
|
27
|
+
Requires-Dist: pytest-asyncio>=0.24; extra == 'dev'
|
|
28
|
+
Requires-Dist: pytest>=8.3; extra == 'dev'
|
|
29
|
+
Description-Content-Type: text/markdown
|
|
30
|
+
|
|
31
|
+
<div align="center">
|
|
32
|
+
|
|
33
|
+
# 🐦⬛ qaas
|
|
34
|
+
|
|
35
|
+
### A multi-agent QA system that finds real defects — and proves it
|
|
36
|
+
|
|
37
|
+
**Reads your application → finds defects → reproduces each with a failing test → files the ticket → fixes it → reviews the fix → verifies it.**
|
|
38
|
+
|
|
39
|
+
[](https://pypi.org/project/qaas-python/)
|
|
40
|
+
[](https://pypi.org/project/qaas-python/)
|
|
41
|
+
[](LICENSE)
|
|
42
|
+
[](https://github.com/allaabdella2-us/qa-multi-agent-system/actions/workflows/ci.yml)
|
|
43
|
+
[](#-contributing)
|
|
44
|
+
[](https://docs.claude.com/en/api/agent-sdk/overview)
|
|
45
|
+
|
|
46
|
+
[Quickstart](#-quickstart-in-60-seconds) · [What it costs](#-what-it-costs) · [Your repo](#-point-it-at-your-repository) · [Jira](#-file-into-jira) · [Architecture](ARCHITECTURE.md)
|
|
47
|
+
|
|
48
|
+
</div>
|
|
49
|
+
|
|
50
|
+
---
|
|
51
|
+
|
|
52
|
+
Most "AI QA" tools generate tests. **This one behaves like a QA team.**
|
|
53
|
+
|
|
54
|
+
Eight agents, each with its own context, tool allowlist and budget, coordinated by
|
|
55
|
+
a state machine that is ordinary Python — because a model cannot enforce a budget
|
|
56
|
+
it is itself spending.
|
|
57
|
+
|
|
58
|
+
```
|
|
59
|
+
DISCOVERY LOOP REMEDIATION LOOP
|
|
60
|
+
┌──────────────────────────────────────┐ ┌───────────────────────────┐
|
|
61
|
+
│ CARTOGRAPHER ─▶ CONDUIT ─┐ │ │ MENDER ─▶ ARBITER │
|
|
62
|
+
│ (system map) SURFACE ─┴─▶ FORGE ─┼─▶ CLERK│ (fix) (review) │
|
|
63
|
+
│ (discover) (repro) │ (file)│ │
|
|
64
|
+
└──────────────────────────────┬───────┘ └──────────┬────────────────┘
|
|
65
|
+
│ │
|
|
66
|
+
▼ ▼
|
|
67
|
+
[ TICKET ] ◀────────────── PROOF (verify)
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
Nothing crosses between the loops except a ticket — which is also the audit trail.
|
|
71
|
+
|
|
72
|
+
---
|
|
73
|
+
|
|
74
|
+
## ✨ Why this one is different
|
|
75
|
+
|
|
76
|
+
| | |
|
|
77
|
+
|---|---|
|
|
78
|
+
| 🧠 **The orchestrator is code, not a prompt** | A model cannot enforce a budget it is spending. Phase ordering, concurrency, retries and the loop breakers live in `conductor.py`. That is also why **649 tests run offline, free, with no API key.** |
|
|
79
|
+
| 🧱 **Every agent is its own `query()`** | Not subagents of a shared parent. Each gets a real context boundary, an enforceable tool allowlist, and its own cost number. |
|
|
80
|
+
| 🔬 **Evidence or it did not happen** | `has_evidence()` and `is_fileable()` are methods on the envelope model, not requests in a prompt. An agent cannot talk its way past them. |
|
|
81
|
+
| 📊 **Measured, not asserted** | A deliberately buggy demo app ships with a golden ledger of **16 seeded defects + 4 planted non-defects**. `qaas score` reports recall *and* precision, so a prompt change has a number attached. |
|
|
82
|
+
| 🔒 **Merge is impossible by construction** | No merge method exists anywhere. `gh pr merge` is refused. Pull requests open as drafts. Shipping stays a human decision. |
|
|
83
|
+
| 🔍 **Every action is on the record** | 28 kinds of ledger event — every tool call, denial, verdict and escalation. `qaas trace` reads it back as a timeline. |
|
|
84
|
+
|
|
85
|
+
---
|
|
86
|
+
|
|
87
|
+
## 🚀 Quickstart in 60 seconds
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
pip install qaas-python
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
cd ~/code/your-app
|
|
95
|
+
qaas init . # inspects the repo, writes + activates a target profile
|
|
96
|
+
qaas validate # config, prompts, allowlists ← no API call
|
|
97
|
+
qaas run --mode pr-check --dry-run # every agent's exact options ← no API call
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Or point it straight at a URL:
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
qaas run --repo https://github.com/you/your-app --dry-run
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
> [!TIP]
|
|
107
|
+
> Nothing above contacts an API. `--dry-run` prints exactly what each agent would
|
|
108
|
+
> receive — model, budget, turn cap, tool allowlist, prompt size.
|
|
109
|
+
|
|
110
|
+
**Auth:** the Claude Code CLI if you are signed in, otherwise `ANTHROPIC_API_KEY`.
|
|
111
|
+
|
|
112
|
+
---
|
|
113
|
+
|
|
114
|
+
## 💰 What it costs
|
|
115
|
+
|
|
116
|
+
> [!IMPORTANT]
|
|
117
|
+
> **Real runs spend real money.** Read this before your first one.
|
|
118
|
+
|
|
119
|
+
Median cost per dispatch, **measured** across real runs — not estimated:
|
|
120
|
+
|
|
121
|
+
| agent | median | what you get |
|
|
122
|
+
|---|--:|---|
|
|
123
|
+
| 🖱️ `SURFACE` | **$3.34** | broken flows, console errors, a11y, forms |
|
|
124
|
+
| 🔧 `MENDER` | **$2.03** | the minimal fix, on a branch |
|
|
125
|
+
| 🔌 `CONDUIT` | **$1.97** | API contract, authz and error-shape defects |
|
|
126
|
+
| 🔨 `FORGE` | **$1.61** | a minimal repro + failing test — **per finding** |
|
|
127
|
+
| ✅ `PROOF` / ⚖️ `ARBITER` | ~$1.00 | verification and adversarial review |
|
|
128
|
+
| 📝 `CLERK` / 🗺️ `CARTOGRAPHER` | ~$0.70 | filing, and the map everything reads |
|
|
129
|
+
|
|
130
|
+
A full discovery run over the demo app found **13 of 16** seeded defects for
|
|
131
|
+
about **$15**. A `fix-cycle` pass costs **$5–7**.
|
|
132
|
+
|
|
133
|
+
> `FORGE` runs **once per finding** in a fresh context, so cost scales with what
|
|
134
|
+
> was found, not with how many agents exist.
|
|
135
|
+
|
|
136
|
+
**The controls are real, not advisory:**
|
|
137
|
+
|
|
138
|
+
- `max_budget_usd` per agent *and* per run mode; the governor checks before every dispatch and **stops the run** rather than overspending.
|
|
139
|
+
- The cap survives a resume — `qaas run --run-id <existing>` carries forward what that run already spent.
|
|
140
|
+
- `qaas validate` refuses a run mode whose agents could outspend its cap.
|
|
141
|
+
- `--dry-run` on everything.
|
|
142
|
+
|
|
143
|
+
---
|
|
144
|
+
|
|
145
|
+
## 🎯 Point it at your repository
|
|
146
|
+
|
|
147
|
+
```bash
|
|
148
|
+
qaas init https://github.com/you/your-app # clones, inspects, writes a profile
|
|
149
|
+
qaas init ~/code/your-app --name your-app # or a local path
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
`init` writes `.qaas/config/targets/<name>.yaml` **and activates it**. Every value
|
|
153
|
+
in it is a **guess you are expected to correct** — it reports what it detected and
|
|
154
|
+
what it could not find.
|
|
155
|
+
|
|
156
|
+
The load-bearing field is `environment.mode`:
|
|
157
|
+
|
|
158
|
+
| mode | meaning | what agents may do |
|
|
159
|
+
|---|---|---|
|
|
160
|
+
| 🔵 `none` | no running instance | read code, schema and spec only |
|
|
161
|
+
| 🟡 `external` | already running (staging, dev server) | read and exercise, **never** reset |
|
|
162
|
+
| 🟢 `compose` | qaas owns the lifecycle | seed, reset, tear down |
|
|
163
|
+
|
|
164
|
+
> [!NOTE]
|
|
165
|
+
> `none` is a perfectly good place to start, and where most first runs against a
|
|
166
|
+
> real repository begin. Findings stay honest about it: an agent that could not
|
|
167
|
+
> observe a behaviour says so and lowers its confidence.
|
|
168
|
+
|
|
169
|
+
**Credentials never live in the profile.** It names environment variables:
|
|
170
|
+
|
|
171
|
+
```yaml
|
|
172
|
+
auth:
|
|
173
|
+
mode: login
|
|
174
|
+
login_endpoint: POST /api/v1/session
|
|
175
|
+
roles:
|
|
176
|
+
admin: { username: qa-admin@example.com, password_env: APP_ADMIN_PASSWORD }
|
|
177
|
+
viewer: { username: qa-viewer@example.com, password_env: APP_VIEWER_PASSWORD }
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
---
|
|
181
|
+
|
|
182
|
+
## 🎫 File into Jira
|
|
183
|
+
|
|
184
|
+
```bash
|
|
185
|
+
export JIRA_BASE_URL=https://you.atlassian.net
|
|
186
|
+
export JIRA_EMAIL=you@example.com
|
|
187
|
+
export JIRA_API_TOKEN=... # an API token, not a password
|
|
188
|
+
export JIRA_PROJECT_KEY=QA
|
|
189
|
+
|
|
190
|
+
QAAS_TRACKER=jira qaas tracker-check # auth, project, permissions, workflow — creates nothing
|
|
191
|
+
QAAS_TRACKER=jira qaas run --mode nightly
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
`tracker-check` validates credentials, confirms the project and issue type exist,
|
|
195
|
+
maps your workflow statuses, and prints the exact JSON it *would* POST. **It
|
|
196
|
+
creates nothing.**
|
|
197
|
+
|
|
198
|
+
- 🔁 **Dedupe across runs** — tickets carry a `qaas-fp-<fingerprint>` label, so the next run recognises an already-filed defect and increments its occurrence count instead of filing again.
|
|
199
|
+
- 🔐 **Security findings are refused** unless `JIRA_SECURITY_PROJECT_KEY` is set. A vulnerability in a project the whole company can read is a disclosure with no undo.
|
|
200
|
+
|
|
201
|
+
> [!TIP]
|
|
202
|
+
> Keep the committed backend `local` — it writes tickets as JSON under
|
|
203
|
+
> `.qaas/tickets/` so you can read what *would* be filed. Switch per shell with
|
|
204
|
+
> `QAAS_TRACKER=jira`.
|
|
205
|
+
|
|
206
|
+
---
|
|
207
|
+
|
|
208
|
+
## 🔌 Bring your own MCP servers
|
|
209
|
+
|
|
210
|
+
Declare them in `system.yaml`. No Python to edit.
|
|
211
|
+
|
|
212
|
+
```yaml
|
|
213
|
+
mcp_servers:
|
|
214
|
+
house-lint:
|
|
215
|
+
type: stdio
|
|
216
|
+
command: ./tools/lint-mcp
|
|
217
|
+
args: ["--strict"]
|
|
218
|
+
env: { LINT_TOKEN: "${ACME_LINT_TOKEN}" } # from the environment, never a literal
|
|
219
|
+
remote-docs:
|
|
220
|
+
type: http
|
|
221
|
+
url: https://mcp.example/v1
|
|
222
|
+
```
|
|
223
|
+
|
|
224
|
+
Then add the name to any agent's `mcp_servers:` list. **Declaring a server grants
|
|
225
|
+
nothing** — an agent receives it only by naming it.
|
|
226
|
+
|
|
227
|
+
```console
|
|
228
|
+
$ qaas validate
|
|
229
|
+
Declared MCP servers
|
|
230
|
+
┃ name ┃ kind ┃ what it runs ┃ used by ┃
|
|
231
|
+
│ house-lint │ stdio │ ./tools/lint-mcp --strict │ CARTOGRAPHER │
|
|
232
|
+
│ remote-docs │ http │ https://mcp.example/v1 │ nobody │
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
> [!WARNING]
|
|
236
|
+
> A server's tools are allowed **wholesale** once an agent names it. qaas checks
|
|
237
|
+
> that the agent declared the server; it cannot inspect what a third-party
|
|
238
|
+
> server's tools actually do. **A server you declare is a server you trust.**
|
|
239
|
+
|
|
240
|
+
There is deliberately **no in-process Python server type** — a module path from a
|
|
241
|
+
config file would mean importing arbitrary code into the process holding your
|
|
242
|
+
Anthropic, Jira and GitHub credentials. Wrap it in a stdio entry point instead.
|
|
243
|
+
|
|
244
|
+
Two limits worth knowing: **6 MCP servers per agent** (tool-selection accuracy
|
|
245
|
+
falls off past ~7), and **MENDER is already at the cap** — the agent people most
|
|
246
|
+
want to extend.
|
|
247
|
+
|
|
248
|
+
---
|
|
249
|
+
|
|
250
|
+
## ✏️ Make the prompts yours
|
|
251
|
+
|
|
252
|
+
Agents are **a prompt plus a YAML file**. Both are yours to change.
|
|
253
|
+
|
|
254
|
+
```bash
|
|
255
|
+
qaas prompts list # which prompt is in force, and where it came from
|
|
256
|
+
qaas prompts eject CONDUIT # copy it to .qaas/prompts/ and edit freely
|
|
257
|
+
qaas prompts diff # what you changed vs. what shipped
|
|
258
|
+
```
|
|
259
|
+
|
|
260
|
+
Prefer **adding** to replacing — drop a `CONDUIT.append.md` beside it:
|
|
261
|
+
|
|
262
|
+
```markdown
|
|
263
|
+
## Our conventions
|
|
264
|
+
- Never file a finding without a curl reproduction.
|
|
265
|
+
- Treat any 500 on a write path as blocker severity.
|
|
266
|
+
```
|
|
267
|
+
|
|
268
|
+
That block is inserted between the agent's prompt and the shared house rules, so
|
|
269
|
+
you keep receiving improvements to the base prompt instead of forking it forever.
|
|
270
|
+
|
|
271
|
+
---
|
|
272
|
+
|
|
273
|
+
## 🔍 Full traceability
|
|
274
|
+
|
|
275
|
+
Every tool call, denial, verdict and escalation is on the record.
|
|
276
|
+
|
|
277
|
+
```console
|
|
278
|
+
$ qaas trace run-20260908T182034-c6ed26
|
|
279
|
+
t+ agent kind detail cost
|
|
280
|
+
0s - run_started mode=nightly agents=[5] budget_usd=40
|
|
281
|
+
0s CARTOGRAPHER agent_started model=claude-sonnet-5
|
|
282
|
+
4s CARTOGRAPHER tool_call ×34 Read×25, Glob×6, ToolSearch×2
|
|
283
|
+
6s CARTOGRAPHER denial tool=Bash reason=Bash is not in CARTOGRAPHER's
|
|
284
|
+
tool allowlist (Read, Grep, Glob).
|
|
285
|
+
146s CARTOGRAPHER system_map version=20260907T233530 sections=[12]
|
|
286
|
+
156s CARTOGRAPHER agent_finished subtype=success num_turns=45 $0.60
|
|
287
|
+
```
|
|
288
|
+
|
|
289
|
+
```bash
|
|
290
|
+
qaas trace <run-id> --agent proof --kind verdict # filter
|
|
291
|
+
qaas trace <run-id> --json # export
|
|
292
|
+
qaas show <run-id> # mode, commit, cost, tickets, escalations
|
|
293
|
+
qaas runs # everything that ever ran
|
|
294
|
+
```
|
|
295
|
+
|
|
296
|
+
Runs are pinned to the **commit of the target** they examined, so a finding can
|
|
297
|
+
be replayed against the tree that produced it.
|
|
298
|
+
|
|
299
|
+
---
|
|
300
|
+
|
|
301
|
+
## 🛡️ Safety rails
|
|
302
|
+
|
|
303
|
+
Enforced in code, not requested in a prompt:
|
|
304
|
+
|
|
305
|
+
| rail | what it does |
|
|
306
|
+
|---|---|
|
|
307
|
+
| 📁 **Path scoping** | Writes checked against that agent's `write_paths`. Most agents cannot write at all. |
|
|
308
|
+
| 🌿 **Branch scoping** | Git writes must match the agent's patterns (`qa/repro/*`, `fix/*`). `main` and force-push refused outright. |
|
|
309
|
+
| ⛔ **Forbidden classes** | Migrations, auth, payment, secrets, infrastructure, CI — stop at a human however small the change looks. |
|
|
310
|
+
| 🎟️ **Ticket rate limit** | Over the per-run cap the call is denied and the conductor escalates rather than filing. |
|
|
311
|
+
| 🧪 **Immutable test** | The agent fixing a defect may not edit the test that defines it. |
|
|
312
|
+
| 🚫 **No filesystem settings** | `setting_sources=[]` — a repository qaas is inspecting cannot inject settings, hooks or MCP servers into the process running it. |
|
|
313
|
+
|
|
314
|
+
Denials return a reason and are logged; they never kill the turn. The agent reads
|
|
315
|
+
the refusal and adapts.
|
|
316
|
+
|
|
317
|
+
---
|
|
318
|
+
|
|
319
|
+
## 📈 Benchmarks, honestly
|
|
320
|
+
|
|
321
|
+
Against the bundled demo app and its golden ledger:
|
|
322
|
+
|
|
323
|
+
Two runs against the demo app, scored automatically:
|
|
324
|
+
|
|
325
|
+
| metric | run A (full loop) | run B (discovery only) |
|
|
326
|
+
|---|--:|--:|
|
|
327
|
+
| 🎯 recall | **81%** — 13 of 16 | **69%** — 11 of 16 |
|
|
328
|
+
| 🔇 precision | **100%** — 0 FP | **92%** — 1 FP |
|
|
329
|
+
| 🏷️ severity agreement | **100%** | **100%** |
|
|
330
|
+
| 💵 cost per accepted finding | $1.12 | $0.64 |
|
|
331
|
+
|
|
332
|
+
**Both numbers are shown on purpose.** A single figure would be the flattering
|
|
333
|
+
one, and it would not survive contact with a second run. These are stochastic
|
|
334
|
+
agents: the two runs did not find the same 11–13 defects — run B caught a
|
|
335
|
+
contrast failure run A missed, and missed three run A found. Expect variance of
|
|
336
|
+
this order.
|
|
337
|
+
|
|
338
|
+
The ledger's `not_defects` section plants **correct-but-suspicious** code, so
|
|
339
|
+
precision is measured rather than assumed.
|
|
340
|
+
|
|
341
|
+
> [!CAUTION]
|
|
342
|
+
> **Treat this as a floor, not a proof.** Seeded defects are easier than real
|
|
343
|
+
> ones and the system was calibrated against them. Two runs is not a sample.
|
|
344
|
+
> These numbers show the loop works end to end and does not spray false
|
|
345
|
+
> positives — not that it will find the hard bug in your codebase.
|
|
346
|
+
|
|
347
|
+
---
|
|
348
|
+
|
|
349
|
+
## 📋 Status
|
|
350
|
+
|
|
351
|
+
Honest about what exists:
|
|
352
|
+
|
|
353
|
+
- ✅ **8 of the 16 agents** in the design are built — CARTOGRAPHER, CONDUIT, SURFACE, FORGE, CLERK, MENDER, ARBITER, PROOF. CONDUCTOR is the Python state machine rather than an agent. **Seven Phase-2 agents are designed, not written.**
|
|
354
|
+
- ✅ The fix loop has closed end to end on a real defect: `NOT_FIXED → MENDER → ARBITER APPROVE → VERIFIED`.
|
|
355
|
+
- ✅ 30 skills, 7 in-process MCP servers, 649 offline tests.
|
|
356
|
+
- ⚠️ Running the bundled demo needs `export CORVID_PASSWORD=password123` — credentials come from the environment, including the demo's.
|
|
357
|
+
|
|
358
|
+
---
|
|
359
|
+
|
|
360
|
+
## 🤝 Contributing
|
|
361
|
+
|
|
362
|
+
```bash
|
|
363
|
+
git clone https://github.com/allaabdella2-us/qa-multi-agent-system
|
|
364
|
+
cd qa-multi-agent-system
|
|
365
|
+
uv venv && uv pip install -e ".[dev]"
|
|
366
|
+
|
|
367
|
+
pytest # 649 tests, offline, free — keep it that way
|
|
368
|
+
pytest -m docker # needs: cd target-app && docker compose up -d
|
|
369
|
+
qaas validate
|
|
370
|
+
```
|
|
371
|
+
|
|
372
|
+
The `llm`, `docker`, `github` and `jira` markers are deselected by default.
|
|
373
|
+
**A CI run that costs money is a CI run people switch off.**
|
|
374
|
+
|
|
375
|
+
> [!IMPORTANT]
|
|
376
|
+
> If you change a seeded defect in `target-app/`, retire its ledger entry with
|
|
377
|
+
> `fixed_in:` in the same commit — a stale ledger silently corrupts every score.
|
|
378
|
+
|
|
379
|
+
New to the codebase? [`ARCHITECTURE.md`](ARCHITECTURE.md) explains the entry
|
|
380
|
+
point, the five phases, what moves between agents, and what the guardrails stop.
|
|
381
|
+
|
|
382
|
+
---
|
|
383
|
+
|
|
384
|
+
<div align="center">
|
|
385
|
+
|
|
386
|
+
**MIT licensed** · [LICENSE](LICENSE) · Built on the [Claude Agent SDK](https://docs.claude.com/en/api/agent-sdk/overview)
|
|
387
|
+
|
|
388
|
+
</div>
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
qaas/cli.py,sha256=L12m7SGyCYNcraUXzu0bCp9A1UtiSF0L6CLzS7xVnVc,65477
|
|
2
|
+
qaas/conductor.py,sha256=Tz0sfw-t4ytglu52FBMxRtq9E_VNpe9kYNWOnp7BYxA,22773
|
|
3
|
+
qaas/config.py,sha256=ZzgK0KUy7MR31w0o7fe1D2tYl--jkZCN_VaWR2kK1Xs,17528
|
|
4
|
+
qaas/discover.py,sha256=L5ejBsYs_s41OWAL-Dhc8VlQ5EWw2hGayWTDzfj05R8,8524
|
|
5
|
+
qaas/envelope.py,sha256=IiqyOy96CHZw2A0pSDWBNIg6yQ2MzfcwkpQWmgMNvzY,9095
|
|
6
|
+
qaas/guardrails.py,sha256=4JIQGFnQpQeIU92c2_E7PQFwXBnQnzolurwp6Zf-Cnk,18797
|
|
7
|
+
qaas/paths.py,sha256=NdW0kDqLXBSASIhp5xFSmMijYPU5tNknKdLj1AUp2to,12703
|
|
8
|
+
qaas/registry.py,sha256=D58UXtYi3uPpW7x6yVDBuEO2azHb2NCqtpYLJleWS54,19664
|
|
9
|
+
qaas/runner.py,sha256=ED0arzgx0_Y6R_mcG6OXqzh3GeSOpqVUNa4NJ0QcRk8,7417
|
|
10
|
+
qaas/scorecard.py,sha256=aUU7g5OtXIW4752A4NxpIZH-OdC8bpNtzJUvCi41JYg,15809
|
|
11
|
+
qaas/sdk_compat.py,sha256=ftE6PK0jZY85zkYqS7UJTGYwRkYDlH4NspiKbKVfVXQ,1678
|
|
12
|
+
qaas/store.py,sha256=aDJmeCog17zo_Dw2Pw_JHTcWnDQlDW3QKzNnUttVvg8,11038
|
|
13
|
+
qaas/target.py,sha256=-wgFDOPm8tCYzideEJV4hdVb1aakLqAM-KJT4Egfgzw,10378
|
|
14
|
+
qaas/tasks.py,sha256=s17e8ykqG61Uthq_pHKYK7BPIY2qzkdFH1aAdVAgCSo,15125
|
|
15
|
+
qaas/trace.py,sha256=V-uFqh1iCYRqgWL3VzDbRdHgAkwLsr1uRFfwwycWZ94,11461
|
|
16
|
+
qaas/adapters/__init__.py,sha256=bw2pqtDqhZGP730gwV28BJ-8TF-rhanwCEjdIizjP6A,882
|
|
17
|
+
qaas/adapters/tracker.py,sha256=K1U7weiA_K2MM58yji3WQn3PEATVqDD9_qcRbrcZwik,54348
|
|
18
|
+
qaas/adapters/vcs.py,sha256=9su-4QLLxR6yTyLZWCJAky9BROJci80M5jV6nGo6pjg,19430
|
|
19
|
+
qaas/defaults/config/system.yaml,sha256=UOKIZ5GHxtUTJEoIDNQR8tcH_0zJ4jeYlzbNYu44LZU,2807
|
|
20
|
+
qaas/defaults/config/agents/arbiter.yaml,sha256=HuPv4E9-p2LW3-bzt3ard9QTJSw98HZYKf-ae6Skd1o,739
|
|
21
|
+
qaas/defaults/config/agents/cartographer.yaml,sha256=9FWySvbwD1bHuMm8CkyaJhmrtgIJrRuRGrli_ZZa_pw,772
|
|
22
|
+
qaas/defaults/config/agents/clerk.yaml,sha256=VgjdqROYr7khipxmou-Ol3Bv-FWj5CZENspUQDA1UB4,768
|
|
23
|
+
qaas/defaults/config/agents/conduit.yaml,sha256=TR9pMOcD5kEzCV-t9oBZ1hS2JklnFPrWsZy9-z7Oikg,728
|
|
24
|
+
qaas/defaults/config/agents/forge.yaml,sha256=WGwkueX0HvaKep-5m2bWR7v6Hx29bs-nuoZbfb4avWA,879
|
|
25
|
+
qaas/defaults/config/agents/mender.yaml,sha256=-HoSJtm5GQSlozgr0O5BOvyibJanYSuJuUYhSs7MRQE,2182
|
|
26
|
+
qaas/defaults/config/agents/proof.yaml,sha256=JbXl2yKUHRv7q-A6MEcz88E3gJn78vNxw6fdmsDbXus,853
|
|
27
|
+
qaas/defaults/config/agents/surface.yaml,sha256=IqmWhQQYArEsNiiRKyw2uTue04CfYreHF9yxJ6z7p_Y,532
|
|
28
|
+
qaas/mcp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
29
|
+
qaas/mcp/context.py,sha256=Z45wQuP0nIZOmM5GTfxs6KHxFckLQ61H4zbsVqiVqr8,2551
|
|
30
|
+
qaas/mcp/contract_diff.py,sha256=efuma0WQSnxHC0NE7adF_8ctCZoovdB4k080QdiEJK8,43942
|
|
31
|
+
qaas/mcp/defect_memory.py,sha256=IFs6s0zYjUv6KvCaaXNgnQn9HAl8y-zDFVCxeWkEwuM,19829
|
|
32
|
+
qaas/mcp/env_control.py,sha256=w19W9KHBqxFidi_et0Z5V8OGy1_b6uTCxUedWEIbk7E,38442
|
|
33
|
+
qaas/mcp/envelope_server.py,sha256=RErNtlt1_SddQNBrtc8rttZu6oVMGgZFC4DqOK9_5Dc,20371
|
|
34
|
+
qaas/mcp/test_runner.py,sha256=QysNYAkHoukJH13nroHLAnt19-m5fiRzvhwfWlMRCg0,30562
|
|
35
|
+
qaas/mcp/tracker.py,sha256=WjlH_cnY50EbXTqo827qBFj820TjoA25fAcpDXuhrCc,18199
|
|
36
|
+
qaas/mcp/vcs.py,sha256=oo9uWZrfFyFy6bYELe_8DTWubhI4VrcfrO3nkT27-i4,20757
|
|
37
|
+
qaas/plugin/.claude-plugin/plugin.json,sha256=XbzYID6aMD3dVPvJxOP9X_0iJR5isqIfKgGkZmn38BU,263
|
|
38
|
+
qaas/plugin/skills/a11y-audit/SKILL.md,sha256=hTpKJkKTYqxkl2kQRqHrsKU1YfhntilzCykkToBnWyY,2634
|
|
39
|
+
qaas/plugin/skills/adversarial-review/SKILL.md,sha256=fu5gz91mEwBx7mNeZBIubsY12LaWw5SdzFgn0XHZLXg,8589
|
|
40
|
+
qaas/plugin/skills/api-surface-extraction/SKILL.md,sha256=5RHGdz4uh0Qez-pcGvCIeSwRAcNDo0KhITqg22pRawA,2349
|
|
41
|
+
qaas/plugin/skills/authz-matrix-check/SKILL.md,sha256=eObh1VTxGrASneSVBKVavIw7uRzWtnAluJRJyT0s8cY,2822
|
|
42
|
+
qaas/plugin/skills/console-error-triage/SKILL.md,sha256=mh5B7tEtLh--2XDdYZWsERE0ojBv8O0greZvr_RzBiI,2221
|
|
43
|
+
qaas/plugin/skills/contract-test-generation/SKILL.md,sha256=n31bPqI88LzOoGvNtyKA20r6qHRUwT69qAoP-LfA19U,2180
|
|
44
|
+
qaas/plugin/skills/dedupe-strategy/SKILL.md,sha256=mMJ1rcQJ88orWwscNQUFI4i3c830n2-d05C4AKF3y7U,2644
|
|
45
|
+
qaas/plugin/skills/environment-pinning/SKILL.md,sha256=Kk8UsGiPhieETq0J7M_Ovf1KAbNeElBl9idNhCjkIP8,1887
|
|
46
|
+
qaas/plugin/skills/error-taxonomy/SKILL.md,sha256=ixnkavgU-br48vQwBhC8elc8nN4lNBQ7mtKzcveg3qY,2574
|
|
47
|
+
qaas/plugin/skills/exploratory-ui-walk/SKILL.md,sha256=NDJn3MQ1E-JgLd09PKhUkCxHo8amFtmEJ2YLYpgDzBI,2454
|
|
48
|
+
qaas/plugin/skills/failing-test-authoring/SKILL.md,sha256=QGPZGDrRR1yvhFk2rokegtDdWtnCTkouFhuMaktwQF8,2158
|
|
49
|
+
qaas/plugin/skills/flake-detection/SKILL.md,sha256=2iDjsfc4xHpgpBfOYx9zbubDoX6XdBJAB0U1W8budOg,2333
|
|
50
|
+
qaas/plugin/skills/form-state-probe/SKILL.md,sha256=IocxBbUr0ATjwWbmTQVN8LU5oBelLJLlLJaGsBRDDQ8,2124
|
|
51
|
+
qaas/plugin/skills/minimal-diff-discipline/SKILL.md,sha256=U4-Lgdgb-oK55a3BZQl-PFh2cSr9aY7auOOVkmyWvJ0,5897
|
|
52
|
+
qaas/plugin/skills/openapi-diff/SKILL.md,sha256=aaJmSGxbLlPy8JI9w09P67p0AzlQbM3lAjYNh3gFi4I,2641
|
|
53
|
+
qaas/plugin/skills/ownership-resolution/SKILL.md,sha256=s1gFLr3ItSKqlN6pN01QaxaXduXxWx5BXdr1CqMRlRY,1897
|
|
54
|
+
qaas/plugin/skills/product-task-graph/SKILL.md,sha256=7Q_3fdx4FCKG9qcFLaukOT6QWzs95OTe5ESFZBCXp7g,2349
|
|
55
|
+
qaas/plugin/skills/regression-risk-scoring/SKILL.md,sha256=mxnL82NjzN9PF6LsRKc8QYJ5tHiJbxWKxb9o1dDCHzc,6147
|
|
56
|
+
qaas/plugin/skills/regression-suite-selection/SKILL.md,sha256=c23I-9DZiP5zRozPitSJe_ekBSkfwL-krbTbnPoUs_U,2241
|
|
57
|
+
qaas/plugin/skills/repo-cartography/SKILL.md,sha256=CIK0Vr9rCuW74og-X8Tsc9bJYW_zOgQS4rGqFP3OMnI,2343
|
|
58
|
+
qaas/plugin/skills/repro-minimisation/SKILL.md,sha256=U-I380BAaiQ-qfXjPUy0j0zTEcNr5WI1iVgj8t7UF4Q,2418
|
|
59
|
+
qaas/plugin/skills/rollback-plan-authoring/SKILL.md,sha256=57ZBLrgtTEy8BmYi0-9FZ1YeoZz6u-aS7e_EbQPDm9g,5639
|
|
60
|
+
qaas/plugin/skills/root-cause-vs-symptom/SKILL.md,sha256=mBWG2HSM5s_Xy5GoXDxgHt0dFbb4gKplKAOPD4T9jOE,7321
|
|
61
|
+
qaas/plugin/skills/routing-rules/SKILL.md,sha256=8v8pJZLGPyM8rXIWpfXpSb5JDCdDNLk5TxLtIt0KSqI,2306
|
|
62
|
+
qaas/plugin/skills/severity-rubric/SKILL.md,sha256=xR9UuClh8tDY9mEGapP6Uagnn-PqQuRX3YsCSZRaki4,3399
|
|
63
|
+
qaas/plugin/skills/test-first-fix/SKILL.md,sha256=c6XVfTU_GYNHBS4wMq3n7JOJS-CZehWdg7O8RwVJexw,5747
|
|
64
|
+
qaas/plugin/skills/test-quality-audit/SKILL.md,sha256=IyICwj8bShyO2Y8q5nrh-_VSTpVHSdWGbiGWkMtKw1c,6435
|
|
65
|
+
qaas/plugin/skills/ticket-writer/SKILL.md,sha256=f2ocjh63erAvTatj15mJ33z51p-ggcl1cWhlVEixvaU,2409
|
|
66
|
+
qaas/plugin/skills/verdict-reporting/SKILL.md,sha256=y3un7-yns_dojK1YYpGaQhXyid44M8Zb8vCwgGSH9gA,1998
|
|
67
|
+
qaas/plugin/skills/verification-protocol/SKILL.md,sha256=6ixQO7GBbyA_fK8CCeyUmPr2Tu_b3avoSG8WfzIyGFs,2455
|
|
68
|
+
qaas/prompts/ARBITER.md,sha256=wFZNLeKx4PMSMmVIPXYObGPvwA1xz6U71sQ0eYc8AKQ,2524
|
|
69
|
+
qaas/prompts/CARTOGRAPHER.md,sha256=7OYfzADgBoaIx39bOM-f3x26lYGDCTli0jbdwlEqsQs,2226
|
|
70
|
+
qaas/prompts/CLERK.md,sha256=iJjOyDEMbDu79e3ZWJerCdftm1FPrFDAkcUGqu6GthE,2106
|
|
71
|
+
qaas/prompts/CONDUIT.md,sha256=_X86pj4DLQw-S5sYJdyCMdRwWdGRwBiNH_j-616ojUM,2253
|
|
72
|
+
qaas/prompts/FORGE.md,sha256=7KvnMrzS1UtucIoE8IbWtAd6BC9J-0YO-e2fSdyptds,2269
|
|
73
|
+
qaas/prompts/MENDER.md,sha256=eot0WsyilfpWkJ9OsEmzeimikHXVSzd3jFKdslSn5GI,2921
|
|
74
|
+
qaas/prompts/PROOF.md,sha256=xJ5pi4O0N1AFq_xZwAeUawJrP6kknRhnkfpDLqgyo2Q,2026
|
|
75
|
+
qaas/prompts/SURFACE.md,sha256=cZ9df27bXXRh39yEkwByyxvHskpcNmh6HvC9C3Jt83Y,2220
|
|
76
|
+
qaas/prompts/_shared.md,sha256=lN_s_rAmakhGuyRuWItC--iyy-Er6ohT_dzGeE_g-fo,2620
|
|
77
|
+
qaas_python-0.1.0.dist-info/METADATA,sha256=GkPiM2k4e3fdPsy4M7LzbAtRbfyI1aFIHSnRqnOSuW8,16788
|
|
78
|
+
qaas_python-0.1.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
|
|
79
|
+
qaas_python-0.1.0.dist-info/entry_points.txt,sha256=6UScfruyhP9N_xGx3tXJGkaoAiB36dkINuKyOH6OkK4,38
|
|
80
|
+
qaas_python-0.1.0.dist-info/licenses/LICENSE,sha256=pHWke5oMtv7PLjIQbN6hRa31J0AKj51VCd5TCTUbbX0,1069
|
|
81
|
+
qaas_python-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Alla Abdella
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|