qaas-python 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {qaas_python-0.1.0 → qaas_python-0.2.0}/.gitignore +6 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/CLAUDE.md +5 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/PKG-INFO +10 -41
- {qaas_python-0.1.0 → qaas_python-0.2.0}/README.md +8 -39
- {qaas_python-0.1.0 → qaas_python-0.2.0}/pyproject.toml +7 -2
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/conductor.py +9 -7
- qaas_python-0.2.0/src/qaas/defaults/config/agents/vault.yaml +21 -0
- qaas_python-0.2.0/src/qaas/defaults/config/agents/warden.yaml +23 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/defaults/config/system.yaml +4 -4
- qaas_python-0.2.0/src/qaas/prompts/VAULT.md +59 -0
- qaas_python-0.2.0/src/qaas/prompts/WARDEN.md +62 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/tasks.py +44 -1
- {qaas_python-0.1.0 → qaas_python-0.2.0}/tests/test_conductor.py +18 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/tests/test_config.py +6 -1
- {qaas_python-0.1.0 → qaas_python-0.2.0}/tests/test_paths.py +4 -1
- {qaas_python-0.1.0 → qaas_python-0.2.0}/tutorial/01-code-structure.md +12 -2
- qaas_python-0.1.0/target-app/cross-tenant-order-detail-leak.png +0 -0
- qaas_python-0.1.0/target-app/new-order-validation-wipes-form.png +0 -0
- qaas_python-0.1.0/target-app/order-placed-banner-but-draft-status.png +0 -0
- qaas_python-0.1.0/target-app/place-order-noop-single-item.png +0 -0
- qaas_python-0.1.0/target-app/signout-secondary-button-contrast.png +0 -0
- qaas_python-0.1.0/target-app/web/dist/assets/index-Cdavr0Eg.css +0 -1
- qaas_python-0.1.0/target-app/web/dist/assets/index-zpKS18v-.js +0 -67
- qaas_python-0.1.0/target-app/web/dist/index.html +0 -13
- {qaas_python-0.1.0 → qaas_python-0.2.0}/ARCHITECTURE.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/BUILD_PLAN.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/LICENSE +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/config/targets/corvid.yaml +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/adapters/__init__.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/adapters/tracker.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/adapters/vcs.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/cli.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/config.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/defaults/config/agents/arbiter.yaml +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/defaults/config/agents/cartographer.yaml +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/defaults/config/agents/clerk.yaml +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/defaults/config/agents/conduit.yaml +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/defaults/config/agents/forge.yaml +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/defaults/config/agents/mender.yaml +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/defaults/config/agents/proof.yaml +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/defaults/config/agents/surface.yaml +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/discover.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/envelope.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/guardrails.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/mcp/__init__.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/mcp/context.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/mcp/contract_diff.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/mcp/defect_memory.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/mcp/env_control.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/mcp/envelope_server.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/mcp/test_runner.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/mcp/tracker.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/mcp/vcs.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/paths.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/plugin/.claude-plugin/plugin.json +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/plugin/skills/a11y-audit/SKILL.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/plugin/skills/adversarial-review/SKILL.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/plugin/skills/api-surface-extraction/SKILL.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/plugin/skills/authz-matrix-check/SKILL.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/plugin/skills/console-error-triage/SKILL.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/plugin/skills/contract-test-generation/SKILL.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/plugin/skills/dedupe-strategy/SKILL.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/plugin/skills/environment-pinning/SKILL.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/plugin/skills/error-taxonomy/SKILL.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/plugin/skills/exploratory-ui-walk/SKILL.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/plugin/skills/failing-test-authoring/SKILL.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/plugin/skills/flake-detection/SKILL.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/plugin/skills/form-state-probe/SKILL.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/plugin/skills/minimal-diff-discipline/SKILL.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/plugin/skills/openapi-diff/SKILL.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/plugin/skills/ownership-resolution/SKILL.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/plugin/skills/product-task-graph/SKILL.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/plugin/skills/regression-risk-scoring/SKILL.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/plugin/skills/regression-suite-selection/SKILL.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/plugin/skills/repo-cartography/SKILL.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/plugin/skills/repro-minimisation/SKILL.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/plugin/skills/rollback-plan-authoring/SKILL.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/plugin/skills/root-cause-vs-symptom/SKILL.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/plugin/skills/routing-rules/SKILL.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/plugin/skills/severity-rubric/SKILL.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/plugin/skills/test-first-fix/SKILL.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/plugin/skills/test-quality-audit/SKILL.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/plugin/skills/ticket-writer/SKILL.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/plugin/skills/verdict-reporting/SKILL.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/plugin/skills/verification-protocol/SKILL.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/prompts/ARBITER.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/prompts/CARTOGRAPHER.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/prompts/CLERK.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/prompts/CONDUIT.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/prompts/FORGE.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/prompts/MENDER.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/prompts/PROOF.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/prompts/SURFACE.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/prompts/_shared.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/registry.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/runner.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/scorecard.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/sdk_compat.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/store.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/target.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/src/qaas/trace.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/CODEOWNERS +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/api/Dockerfile +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/api/app/__init__.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/api/app/auth.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/api/app/config.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/api/app/db.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/api/app/errors.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/api/app/main.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/api/app/models.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/api/app/routes/__init__.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/api/app/routes/auth.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/api/app/routes/invoices.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/api/app/routes/orders.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/api/app/routes/stream.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/api/app/schemas.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/api/migrations/001_init.sql +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/api/pyproject.toml +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/api/seed/fixtures.sql +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/defects.yaml +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/docker-compose.yml +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/openapi.yaml +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/web/.gitignore +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/web/Dockerfile +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/web/index.html +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/web/package-lock.json +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/web/package.json +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/web/src/api.ts +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/web/src/components/Button.tsx +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/web/src/components/Layout.tsx +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/web/src/components/SearchInput.tsx +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/web/src/main.tsx +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/web/src/routes/CheckoutReview.tsx +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/web/src/routes/Login.tsx +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/web/src/routes/NewOrder.tsx +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/web/src/routes/OrderDetail.tsx +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/web/src/routes/OrdersList.tsx +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/web/src/styles.css +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/web/tsconfig.json +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/target-app/web/vite.config.ts +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/tests/adapters/test_github_vcs.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/tests/adapters/test_jira_tracker.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/tests/conftest.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/tests/mcp/conftest.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/tests/mcp/test_contract_diff.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/tests/mcp/test_defect_memory.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/tests/mcp/test_env_control.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/tests/mcp/test_test_runner.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/tests/mcp/test_tracker.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/tests/mcp/test_vcs.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/tests/support.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/tests/target_app/test_seeded_defects.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/tests/test_cli.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/tests/test_discover.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/tests/test_envelope.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/tests/test_guardrails.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/tests/test_hooks_and_skills.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/tests/test_prompt_overrides.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/tests/test_registry.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/tests/test_scorecard.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/tests/test_skills_actually_load.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/tests/test_store.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/tests/test_target_profile_is_honoured.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/tests/test_trace.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/tests/test_user_mcp_servers.py +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/tutorial/02-how-agents-communicate.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/tutorial/03-skills-and-hooks.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/tutorial/04-mcp-servers.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/tutorial/05-prompt-configuration.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/tutorial/06-jira-integration.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/tutorial/07-github-actions.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/tutorial/08-packaging-and-publishing.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/tutorial/09-guardrails-and-safety.md +0 -0
- {qaas_python-0.1.0 → qaas_python-0.2.0}/tutorial/README.md +0 -0
|
@@ -80,6 +80,11 @@ adding an agent as a sign something is wrong. Constraints enforced in
|
|
|
80
80
|
`config.py`: at most 6 MCP servers per agent (§5.3, tool-selection accuracy),
|
|
81
81
|
and every `must_call` tool must name a server the agent actually has.
|
|
82
82
|
|
|
83
|
+
Adding a discovery agent is a prompt file plus a YAML file and no Python --
|
|
84
|
+
`_phase_discover` falls back to `tasks.discovery` for anything without a
|
|
85
|
+
bespoke builder. VAULT and WARDEN were added that way and found that it was
|
|
86
|
+
not true before them.
|
|
87
|
+
|
|
83
88
|
`prompts/_shared.md` is appended to every agent prompt — house rules go there,
|
|
84
89
|
not copy-pasted into six prompts. Prompts resolve through `Workspace.prompt_dirs`
|
|
85
90
|
(`.qaas/prompts/` beats the packaged copy), **file by file and independently**, so
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: qaas-python
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: A multi-agent QA system: finds real defects, reproduces them, files tickets, fixes them, and proves the fix
|
|
5
5
|
Project-URL: Homepage, https://github.com/allaabdella2-us/qa-multi-agent-system
|
|
6
6
|
Project-URL: Repository, https://github.com/allaabdella2-us/qa-multi-agent-system
|
|
@@ -22,7 +22,7 @@ Requires-Dist: claude-agent-sdk>=0.2.127
|
|
|
22
22
|
Requires-Dist: pydantic>=2.9
|
|
23
23
|
Requires-Dist: pyyaml>=6.0
|
|
24
24
|
Requires-Dist: rich>=13.9
|
|
25
|
-
Requires-Dist: typer>=0.
|
|
25
|
+
Requires-Dist: typer>=0.16
|
|
26
26
|
Provides-Extra: dev
|
|
27
27
|
Requires-Dist: pytest-asyncio>=0.24; extra == 'dev'
|
|
28
28
|
Requires-Dist: pytest>=8.3; extra == 'dev'
|
|
@@ -43,7 +43,7 @@ Description-Content-Type: text/markdown
|
|
|
43
43
|
[](#-contributing)
|
|
44
44
|
[](https://docs.claude.com/en/api/agent-sdk/overview)
|
|
45
45
|
|
|
46
|
-
[Quickstart](#-quickstart-in-60-seconds) · [
|
|
46
|
+
[Quickstart](#-quickstart-in-60-seconds) · [Your repo](#-point-it-at-your-repository) · [Jira](#-file-into-jira) · [Architecture](ARCHITECTURE.md)
|
|
47
47
|
|
|
48
48
|
</div>
|
|
49
49
|
|
|
@@ -51,7 +51,7 @@ Description-Content-Type: text/markdown
|
|
|
51
51
|
|
|
52
52
|
Most "AI QA" tools generate tests. **This one behaves like a QA team.**
|
|
53
53
|
|
|
54
|
-
|
|
54
|
+
Ten agents, each with its own context, tool allowlist and budget, coordinated by
|
|
55
55
|
a state machine that is ordinary Python — because a model cannot enforce a budget
|
|
56
56
|
it is itself spending.
|
|
57
57
|
|
|
@@ -111,37 +111,6 @@ qaas run --repo https://github.com/you/your-app --dry-run
|
|
|
111
111
|
|
|
112
112
|
---
|
|
113
113
|
|
|
114
|
-
## 💰 What it costs
|
|
115
|
-
|
|
116
|
-
> [!IMPORTANT]
|
|
117
|
-
> **Real runs spend real money.** Read this before your first one.
|
|
118
|
-
|
|
119
|
-
Median cost per dispatch, **measured** across real runs — not estimated:
|
|
120
|
-
|
|
121
|
-
| agent | median | what you get |
|
|
122
|
-
|---|--:|---|
|
|
123
|
-
| 🖱️ `SURFACE` | **$3.34** | broken flows, console errors, a11y, forms |
|
|
124
|
-
| 🔧 `MENDER` | **$2.03** | the minimal fix, on a branch |
|
|
125
|
-
| 🔌 `CONDUIT` | **$1.97** | API contract, authz and error-shape defects |
|
|
126
|
-
| 🔨 `FORGE` | **$1.61** | a minimal repro + failing test — **per finding** |
|
|
127
|
-
| ✅ `PROOF` / ⚖️ `ARBITER` | ~$1.00 | verification and adversarial review |
|
|
128
|
-
| 📝 `CLERK` / 🗺️ `CARTOGRAPHER` | ~$0.70 | filing, and the map everything reads |
|
|
129
|
-
|
|
130
|
-
A full discovery run over the demo app found **13 of 16** seeded defects for
|
|
131
|
-
about **$15**. A `fix-cycle` pass costs **$5–7**.
|
|
132
|
-
|
|
133
|
-
> `FORGE` runs **once per finding** in a fresh context, so cost scales with what
|
|
134
|
-
> was found, not with how many agents exist.
|
|
135
|
-
|
|
136
|
-
**The controls are real, not advisory:**
|
|
137
|
-
|
|
138
|
-
- `max_budget_usd` per agent *and* per run mode; the governor checks before every dispatch and **stops the run** rather than overspending.
|
|
139
|
-
- The cap survives a resume — `qaas run --run-id <existing>` carries forward what that run already spent.
|
|
140
|
-
- `qaas validate` refuses a run mode whose agents could outspend its cap.
|
|
141
|
-
- `--dry-run` on everything.
|
|
142
|
-
|
|
143
|
-
---
|
|
144
|
-
|
|
145
114
|
## 🎯 Point it at your repository
|
|
146
115
|
|
|
147
116
|
```bash
|
|
@@ -276,20 +245,20 @@ Every tool call, denial, verdict and escalation is on the record.
|
|
|
276
245
|
|
|
277
246
|
```console
|
|
278
247
|
$ qaas trace run-20260908T182034-c6ed26
|
|
279
|
-
t+ agent kind detail
|
|
280
|
-
0s - run_started mode=nightly agents=[
|
|
248
|
+
t+ agent kind detail
|
|
249
|
+
0s - run_started mode=nightly agents=[7]
|
|
281
250
|
0s CARTOGRAPHER agent_started model=claude-sonnet-5
|
|
282
251
|
4s CARTOGRAPHER tool_call ×34 Read×25, Glob×6, ToolSearch×2
|
|
283
252
|
6s CARTOGRAPHER denial tool=Bash reason=Bash is not in CARTOGRAPHER's
|
|
284
253
|
tool allowlist (Read, Grep, Glob).
|
|
285
254
|
146s CARTOGRAPHER system_map version=20260907T233530 sections=[12]
|
|
286
|
-
156s CARTOGRAPHER agent_finished subtype=success num_turns=45
|
|
255
|
+
156s CARTOGRAPHER agent_finished subtype=success num_turns=45
|
|
287
256
|
```
|
|
288
257
|
|
|
289
258
|
```bash
|
|
290
259
|
qaas trace <run-id> --agent proof --kind verdict # filter
|
|
291
260
|
qaas trace <run-id> --json # export
|
|
292
|
-
qaas show <run-id> # mode, commit,
|
|
261
|
+
qaas show <run-id> # mode, commit, tickets, escalations
|
|
293
262
|
qaas runs # everything that ever ran
|
|
294
263
|
```
|
|
295
264
|
|
|
@@ -327,7 +296,6 @@ Two runs against the demo app, scored automatically:
|
|
|
327
296
|
| 🎯 recall | **81%** — 13 of 16 | **69%** — 11 of 16 |
|
|
328
297
|
| 🔇 precision | **100%** — 0 FP | **92%** — 1 FP |
|
|
329
298
|
| 🏷️ severity agreement | **100%** | **100%** |
|
|
330
|
-
| 💵 cost per accepted finding | $1.12 | $0.64 |
|
|
331
299
|
|
|
332
300
|
**Both numbers are shown on purpose.** A single figure would be the flattering
|
|
333
301
|
one, and it would not survive contact with a second run. These are stochastic
|
|
@@ -350,7 +318,8 @@ precision is measured rather than assumed.
|
|
|
350
318
|
|
|
351
319
|
Honest about what exists:
|
|
352
320
|
|
|
353
|
-
- ✅ **
|
|
321
|
+
- ✅ **10 of the 16 agents** in the design are built — CARTOGRAPHER, CONDUIT, SURFACE, VAULT, WARDEN, FORGE, CLERK, MENDER, ARBITER, PROOF. CONDUCTOR is the Python state machine rather than an agent. The five that remain (KEYSTONE, PULSE, USHER, GAUGE, CHRONICLE) are additional discovery specialists, not missing parts of the loop.
|
|
322
|
+
- ✅ **Adding an agent needs a prompt file and a YAML file — no Python.** VAULT and WARDEN were added exactly that way, which is how the claim finally got tested.
|
|
354
323
|
- ✅ The fix loop has closed end to end on a real defect: `NOT_FIXED → MENDER → ARBITER APPROVE → VERIFIED`.
|
|
355
324
|
- ✅ 30 skills, 7 in-process MCP servers, 649 offline tests.
|
|
356
325
|
- ⚠️ Running the bundled demo needs `export CORVID_PASSWORD=password123` — credentials come from the environment, including the demo's.
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
[](#-contributing)
|
|
14
14
|
[](https://docs.claude.com/en/api/agent-sdk/overview)
|
|
15
15
|
|
|
16
|
-
[Quickstart](#-quickstart-in-60-seconds) · [
|
|
16
|
+
[Quickstart](#-quickstart-in-60-seconds) · [Your repo](#-point-it-at-your-repository) · [Jira](#-file-into-jira) · [Architecture](ARCHITECTURE.md)
|
|
17
17
|
|
|
18
18
|
</div>
|
|
19
19
|
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
|
|
22
22
|
Most "AI QA" tools generate tests. **This one behaves like a QA team.**
|
|
23
23
|
|
|
24
|
-
|
|
24
|
+
Ten agents, each with its own context, tool allowlist and budget, coordinated by
|
|
25
25
|
a state machine that is ordinary Python — because a model cannot enforce a budget
|
|
26
26
|
it is itself spending.
|
|
27
27
|
|
|
@@ -81,37 +81,6 @@ qaas run --repo https://github.com/you/your-app --dry-run
|
|
|
81
81
|
|
|
82
82
|
---
|
|
83
83
|
|
|
84
|
-
## 💰 What it costs
|
|
85
|
-
|
|
86
|
-
> [!IMPORTANT]
|
|
87
|
-
> **Real runs spend real money.** Read this before your first one.
|
|
88
|
-
|
|
89
|
-
Median cost per dispatch, **measured** across real runs — not estimated:
|
|
90
|
-
|
|
91
|
-
| agent | median | what you get |
|
|
92
|
-
|---|--:|---|
|
|
93
|
-
| 🖱️ `SURFACE` | **$3.34** | broken flows, console errors, a11y, forms |
|
|
94
|
-
| 🔧 `MENDER` | **$2.03** | the minimal fix, on a branch |
|
|
95
|
-
| 🔌 `CONDUIT` | **$1.97** | API contract, authz and error-shape defects |
|
|
96
|
-
| 🔨 `FORGE` | **$1.61** | a minimal repro + failing test — **per finding** |
|
|
97
|
-
| ✅ `PROOF` / ⚖️ `ARBITER` | ~$1.00 | verification and adversarial review |
|
|
98
|
-
| 📝 `CLERK` / 🗺️ `CARTOGRAPHER` | ~$0.70 | filing, and the map everything reads |
|
|
99
|
-
|
|
100
|
-
A full discovery run over the demo app found **13 of 16** seeded defects for
|
|
101
|
-
about **$15**. A `fix-cycle` pass costs **$5–7**.
|
|
102
|
-
|
|
103
|
-
> `FORGE` runs **once per finding** in a fresh context, so cost scales with what
|
|
104
|
-
> was found, not with how many agents exist.
|
|
105
|
-
|
|
106
|
-
**The controls are real, not advisory:**
|
|
107
|
-
|
|
108
|
-
- `max_budget_usd` per agent *and* per run mode; the governor checks before every dispatch and **stops the run** rather than overspending.
|
|
109
|
-
- The cap survives a resume — `qaas run --run-id <existing>` carries forward what that run already spent.
|
|
110
|
-
- `qaas validate` refuses a run mode whose agents could outspend its cap.
|
|
111
|
-
- `--dry-run` on everything.
|
|
112
|
-
|
|
113
|
-
---
|
|
114
|
-
|
|
115
84
|
## 🎯 Point it at your repository
|
|
116
85
|
|
|
117
86
|
```bash
|
|
@@ -246,20 +215,20 @@ Every tool call, denial, verdict and escalation is on the record.
|
|
|
246
215
|
|
|
247
216
|
```console
|
|
248
217
|
$ qaas trace run-20260908T182034-c6ed26
|
|
249
|
-
t+ agent kind detail
|
|
250
|
-
0s - run_started mode=nightly agents=[
|
|
218
|
+
t+ agent kind detail
|
|
219
|
+
0s - run_started mode=nightly agents=[7]
|
|
251
220
|
0s CARTOGRAPHER agent_started model=claude-sonnet-5
|
|
252
221
|
4s CARTOGRAPHER tool_call ×34 Read×25, Glob×6, ToolSearch×2
|
|
253
222
|
6s CARTOGRAPHER denial tool=Bash reason=Bash is not in CARTOGRAPHER's
|
|
254
223
|
tool allowlist (Read, Grep, Glob).
|
|
255
224
|
146s CARTOGRAPHER system_map version=20260907T233530 sections=[12]
|
|
256
|
-
156s CARTOGRAPHER agent_finished subtype=success num_turns=45
|
|
225
|
+
156s CARTOGRAPHER agent_finished subtype=success num_turns=45
|
|
257
226
|
```
|
|
258
227
|
|
|
259
228
|
```bash
|
|
260
229
|
qaas trace <run-id> --agent proof --kind verdict # filter
|
|
261
230
|
qaas trace <run-id> --json # export
|
|
262
|
-
qaas show <run-id> # mode, commit,
|
|
231
|
+
qaas show <run-id> # mode, commit, tickets, escalations
|
|
263
232
|
qaas runs # everything that ever ran
|
|
264
233
|
```
|
|
265
234
|
|
|
@@ -297,7 +266,6 @@ Two runs against the demo app, scored automatically:
|
|
|
297
266
|
| 🎯 recall | **81%** — 13 of 16 | **69%** — 11 of 16 |
|
|
298
267
|
| 🔇 precision | **100%** — 0 FP | **92%** — 1 FP |
|
|
299
268
|
| 🏷️ severity agreement | **100%** | **100%** |
|
|
300
|
-
| 💵 cost per accepted finding | $1.12 | $0.64 |
|
|
301
269
|
|
|
302
270
|
**Both numbers are shown on purpose.** A single figure would be the flattering
|
|
303
271
|
one, and it would not survive contact with a second run. These are stochastic
|
|
@@ -320,7 +288,8 @@ precision is measured rather than assumed.
|
|
|
320
288
|
|
|
321
289
|
Honest about what exists:
|
|
322
290
|
|
|
323
|
-
- ✅ **
|
|
291
|
+
- ✅ **10 of the 16 agents** in the design are built — CARTOGRAPHER, CONDUIT, SURFACE, VAULT, WARDEN, FORGE, CLERK, MENDER, ARBITER, PROOF. CONDUCTOR is the Python state machine rather than an agent. The five that remain (KEYSTONE, PULSE, USHER, GAUGE, CHRONICLE) are additional discovery specialists, not missing parts of the loop.
|
|
292
|
+
- ✅ **Adding an agent needs a prompt file and a YAML file — no Python.** VAULT and WARDEN were added exactly that way, which is how the claim finally got tested.
|
|
324
293
|
- ✅ The fix loop has closed end to end on a real defect: `NOT_FIXED → MENDER → ARBITER APPROVE → VERIFIED`.
|
|
325
294
|
- ✅ 30 skills, 7 in-process MCP servers, 649 offline tests.
|
|
326
295
|
- ⚠️ Running the bundled demo needs `export CORVID_PASSWORD=password123` — credentials come from the environment, including the demo's.
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
# Distribution name is `qaas-python` (`qaas` was taken); the import package and
|
|
3
3
|
# the CLI are both `qaas`.
|
|
4
4
|
name = "qaas-python"
|
|
5
|
-
version = "0.
|
|
5
|
+
version = "0.2.0"
|
|
6
6
|
description = "A multi-agent QA system: finds real defects, reproduces them, files tickets, fixes them, and proves the fix"
|
|
7
7
|
readme = "README.md"
|
|
8
8
|
license = "MIT"
|
|
@@ -23,7 +23,12 @@ dependencies = [
|
|
|
23
23
|
"claude-agent-sdk>=0.2.127",
|
|
24
24
|
"pydantic>=2.9",
|
|
25
25
|
"pyyaml>=6.0",
|
|
26
|
-
|
|
26
|
+
# typer>=0.16 is not cosmetic: click 8.2 changed Parameter.make_metavar()
|
|
27
|
+
# to require a ctx argument, and typer 0.15 calls it without one. The pair
|
|
28
|
+
# `typer 0.15.3` + `click 8.2.0` resolves happily and then crashes on
|
|
29
|
+
# `qaas <command> --help` with a TypeError -- which is how a user found it,
|
|
30
|
+
# on a package installed from PyPI.
|
|
31
|
+
"typer>=0.16",
|
|
27
32
|
"rich>=13.9",
|
|
28
33
|
]
|
|
29
34
|
|
|
@@ -272,18 +272,20 @@ class Conductor:
|
|
|
272
272
|
if not discovery:
|
|
273
273
|
return
|
|
274
274
|
|
|
275
|
+
# CONDUIT and SURFACE keep bespoke tasks because they name tools only
|
|
276
|
+
# they have. Everything else gets the generic discovery task, which is
|
|
277
|
+
# what makes "a new agent is a prompt plus a YAML" true: this used to be
|
|
278
|
+
# a closed dict, so a new discovery agent was skipped with `no task
|
|
279
|
+
# builder` -- it validated, it assembled, it showed up in `--dry-run`,
|
|
280
|
+
# and then it silently did nothing.
|
|
275
281
|
builders = {
|
|
276
|
-
"CONDUIT": lambda: tasks.conduit(self.config, mode),
|
|
277
|
-
"SURFACE": lambda: tasks.surface(self.config, mode),
|
|
282
|
+
"CONDUIT": lambda spec: tasks.conduit(self.config, mode),
|
|
283
|
+
"SURFACE": lambda spec: tasks.surface(self.config, mode),
|
|
278
284
|
}
|
|
279
285
|
jobs = [
|
|
280
|
-
(spec, builders
|
|
286
|
+
(spec, builders.get(spec.name, lambda sp: tasks.discovery(self.config, mode, sp))(spec))
|
|
281
287
|
for spec in discovery
|
|
282
|
-
if spec.name in builders
|
|
283
288
|
]
|
|
284
|
-
unknown = [s.name for s in discovery if s.name not in builders]
|
|
285
|
-
if unknown:
|
|
286
|
-
store.log("skipped", reason="no task builder", agents=unknown)
|
|
287
289
|
|
|
288
290
|
await self._gather(jobs, store, budget, report, map_version, self.config.run_modes[mode].max_concurrency)
|
|
289
291
|
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
name: VAULT
|
|
2
|
+
layer: discovery
|
|
3
|
+
role: >
|
|
4
|
+
Database and data-integrity analyst. Finds schema constraints the application
|
|
5
|
+
assumes but the database does not enforce, migrations that lose or corrupt
|
|
6
|
+
data, missing indexes on paths the code queries, and cross-tenant reads that
|
|
7
|
+
the ORM makes easy to write. Reports what the schema actually says, never what
|
|
8
|
+
the model layer claims.
|
|
9
|
+
prompt: VAULT.md
|
|
10
|
+
model: claude-opus-5
|
|
11
|
+
effort: high
|
|
12
|
+
max_turns: 60
|
|
13
|
+
max_budget_usd: 3.0
|
|
14
|
+
mcp_servers: [envelope, env_control, defect_memory]
|
|
15
|
+
builtin_tools: [Read, Grep, Glob]
|
|
16
|
+
policy: {}
|
|
17
|
+
|
|
18
|
+
skills: [authz-matrix-check, environment-pinning, severity-rubric, repro-minimisation]
|
|
19
|
+
|
|
20
|
+
# No must_call: finding nothing is a valid outcome for a discovery agent, and
|
|
21
|
+
# requiring an emission would manufacture findings to satisfy it.
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
name: WARDEN
|
|
2
|
+
layer: discovery
|
|
3
|
+
role: >
|
|
4
|
+
Security and dependency auditor. Finds missing authorization, secrets committed
|
|
5
|
+
to the repository, dependencies with known advisories, and error paths that
|
|
6
|
+
leak internals to a caller. Reports a concrete exploit path or lowers its
|
|
7
|
+
confidence -- a security finding without one is a guess wearing a severity.
|
|
8
|
+
prompt: WARDEN.md
|
|
9
|
+
model: claude-opus-5
|
|
10
|
+
effort: high
|
|
11
|
+
max_turns: 60
|
|
12
|
+
# Measured: WARDEN exhausted $3.00 on its first real run against the demo app
|
|
13
|
+
# and was killed mid-audit. Building the endpoint-by-role matrix and actually
|
|
14
|
+
# impersonating each role costs more than reading a spec does.
|
|
15
|
+
max_budget_usd: 5.0
|
|
16
|
+
mcp_servers: [envelope, env_control, defect_memory]
|
|
17
|
+
builtin_tools: [Read, Grep, Glob]
|
|
18
|
+
policy: {}
|
|
19
|
+
|
|
20
|
+
skills: [authz-matrix-check, error-taxonomy, severity-rubric, routing-rules, repro-minimisation]
|
|
21
|
+
|
|
22
|
+
# No must_call: see VAULT. Also: an auditor that must report something will
|
|
23
|
+
# report something, and security noise is the fastest way to be ignored.
|
|
@@ -33,11 +33,11 @@ run_modes:
|
|
|
33
33
|
|
|
34
34
|
nightly:
|
|
35
35
|
trigger: cron
|
|
36
|
-
agents: [CARTOGRAPHER, CONDUIT, SURFACE, FORGE, CLERK]
|
|
36
|
+
agents: [CARTOGRAPHER, CONDUIT, SURFACE, VAULT, WARDEN, FORGE, CLERK]
|
|
37
37
|
# FORGE runs once per finding, so the deep sweep's budget scales with how
|
|
38
38
|
# much discovery found, not with the number of agents. Measured: discovery
|
|
39
39
|
# ~$6, then roughly $1-2 per finding reproduced.
|
|
40
|
-
max_budget_usd:
|
|
40
|
+
max_budget_usd: 50.0
|
|
41
41
|
max_wall_clock_s: 7200
|
|
42
42
|
max_concurrency: 3
|
|
43
43
|
|
|
@@ -63,7 +63,7 @@ run_modes:
|
|
|
63
63
|
# expensive mode in the system and the only one that closes the loop.
|
|
64
64
|
full-loop:
|
|
65
65
|
trigger: on_demand
|
|
66
|
-
agents: [CARTOGRAPHER, CONDUIT, SURFACE, FORGE, CLERK, MENDER, ARBITER, PROOF]
|
|
67
|
-
max_budget_usd:
|
|
66
|
+
agents: [CARTOGRAPHER, CONDUIT, SURFACE, VAULT, WARDEN, FORGE, CLERK, MENDER, ARBITER, PROOF]
|
|
67
|
+
max_budget_usd: 70.0
|
|
68
68
|
max_wall_clock_s: 10800
|
|
69
69
|
max_concurrency: 3
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
You are VAULT, the database and data-integrity analyst.
|
|
2
|
+
|
|
3
|
+
## Your domain
|
|
4
|
+
|
|
5
|
+
The schema, and the distance between what it enforces and what the application
|
|
6
|
+
assumes. Application code is full of invariants nobody wrote down; your job is to
|
|
7
|
+
find the ones the database will not hold up.
|
|
8
|
+
|
|
9
|
+
Detect:
|
|
10
|
+
|
|
11
|
+
- **Constraints the code assumes and the schema does not enforce** — a field the
|
|
12
|
+
application treats as required with no `NOT NULL`, a relationship it treats as
|
|
13
|
+
unique with no unique index, an enum validated only in the model layer.
|
|
14
|
+
- **Missing foreign keys**, or ones declared without a delete rule, so a parent
|
|
15
|
+
row can leave orphans behind.
|
|
16
|
+
- **Cross-tenant reads** — a query filtered by id but not by the owning
|
|
17
|
+
organisation, on a table that has an owner column. The ORM makes this easy to
|
|
18
|
+
write and hard to see.
|
|
19
|
+
- **Migrations that lose or corrupt data** — a column dropped and re-added, a type
|
|
20
|
+
narrowed without a backfill, a `NOT NULL` added without a default over existing
|
|
21
|
+
rows.
|
|
22
|
+
- **Indexes the query patterns need and the schema lacks** — a column filtered or
|
|
23
|
+
joined on in application code with no index behind it. Say which query, not
|
|
24
|
+
just which column.
|
|
25
|
+
- **Seed and fixture drift** — fixtures that no longer satisfy the constraints the
|
|
26
|
+
migrations now declare.
|
|
27
|
+
|
|
28
|
+
## How you work
|
|
29
|
+
|
|
30
|
+
1. Read the system map for the schema snapshot and the route inventory. Do not
|
|
31
|
+
rediscover them.
|
|
32
|
+
2. Read the migrations in order. The current schema is the sum of them, and a
|
|
33
|
+
defect is often visible only in the sequence — a constraint added, then
|
|
34
|
+
dropped two migrations later to make a deploy pass.
|
|
35
|
+
3. Read the model and query layer and compare its assumptions against what the
|
|
36
|
+
schema actually declares. The gap between the two is your finding.
|
|
37
|
+
4. Where an environment is available, confirm the behaviour rather than inferring
|
|
38
|
+
it: insert the row the code believes is impossible, and see whether the
|
|
39
|
+
database refuses it.
|
|
40
|
+
5. Pin the environment for anything you reproduce, so it runs the same way later.
|
|
41
|
+
|
|
42
|
+
## What counts as evidence
|
|
43
|
+
|
|
44
|
+
The schema text, the migration, and the query. A finding that says "this column
|
|
45
|
+
should be indexed" without naming the query that scans it is an opinion. A
|
|
46
|
+
finding that says "this insert succeeds and the model layer says it cannot" with
|
|
47
|
+
the statement and the response is a defect.
|
|
48
|
+
|
|
49
|
+
Where you could not observe the behaviour — no reachable database, no fixture
|
|
50
|
+
that reaches the path — say so plainly and lower your confidence. An honest
|
|
51
|
+
`unattempted` reproduction is worth more than a confident guess, because the next
|
|
52
|
+
agent will treat your confidence as real.
|
|
53
|
+
|
|
54
|
+
## What is not yours
|
|
55
|
+
|
|
56
|
+
The HTTP surface is CONDUIT's, the UI is SURFACE's, and dependency advisories are
|
|
57
|
+
WARDEN's. A cross-tenant read is yours when the defect is in the query, and
|
|
58
|
+
CONDUIT's when the defect is in the missing authorization check. If both are true,
|
|
59
|
+
report the one you can evidence.
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
You are WARDEN, the security and dependency auditor.
|
|
2
|
+
|
|
3
|
+
## Your domain
|
|
4
|
+
|
|
5
|
+
The things that let someone do what they should not be able to do. You are the
|
|
6
|
+
agent whose findings carry the most weight and therefore cost the most when they
|
|
7
|
+
are wrong.
|
|
8
|
+
|
|
9
|
+
Detect:
|
|
10
|
+
|
|
11
|
+
- **Missing or wrong authorization** — an endpoint that mutates or reads data
|
|
12
|
+
without checking the caller's role, or that checks authentication and calls it
|
|
13
|
+
authorization. The presence of an auth dependency is not evidence that access
|
|
14
|
+
is checked.
|
|
15
|
+
- **Cross-tenant access** — one organisation's data reachable by another's user.
|
|
16
|
+
- **Secrets in the repository** — keys, tokens, passwords and connection strings
|
|
17
|
+
in source, fixtures, CI config or committed environment files.
|
|
18
|
+
- **Dependencies with known advisories**, and dependencies pinned to a version
|
|
19
|
+
behind a security release.
|
|
20
|
+
- **Internal detail leaking to a caller** — stack traces, SQL, file paths, library
|
|
21
|
+
versions in an error response.
|
|
22
|
+
- **Mass assignment** — a handler that accepts fields the client should not
|
|
23
|
+
control, such as a role, a price, or a status.
|
|
24
|
+
- **Weak or absent rate limiting** on authentication and password-reset paths.
|
|
25
|
+
|
|
26
|
+
## How you work
|
|
27
|
+
|
|
28
|
+
1. Read the system map for the route inventory and the role matrix. Do not
|
|
29
|
+
rediscover them.
|
|
30
|
+
2. Build the endpoint-by-role matrix and look for the holes, rather than reading
|
|
31
|
+
handlers in file order and hoping to notice.
|
|
32
|
+
3. Where an environment is available, **demonstrate the access** — impersonate the
|
|
33
|
+
lower-privilege role and make the call. A refusal you predicted and a refusal
|
|
34
|
+
you observed are different findings.
|
|
35
|
+
4. For dependencies, name the advisory and the version that fixes it.
|
|
36
|
+
|
|
37
|
+
## The bar for a security finding
|
|
38
|
+
|
|
39
|
+
**A concrete exploit path, or lower your confidence.** Say which role, which
|
|
40
|
+
endpoint, which field, and what they get. "This endpoint may be missing an
|
|
41
|
+
authorization check" is a note to yourself; "a viewer can POST
|
|
42
|
+
/v1/orders/3/refund and it succeeds" is a finding.
|
|
43
|
+
|
|
44
|
+
This matters more here than anywhere else in the system. A security finding is
|
|
45
|
+
routed to a restricted project, wakes people up, and is read as urgent. A false
|
|
46
|
+
one spends that credibility, and the next real finding is read more slowly. If
|
|
47
|
+
you cannot evidence it, report it with the confidence it actually deserves and
|
|
48
|
+
say what you could not test.
|
|
49
|
+
|
|
50
|
+
## Routing
|
|
51
|
+
|
|
52
|
+
Security findings are routed to a restricted project, and the tracker will
|
|
53
|
+
**refuse** to file one if no restricted project is configured rather than filing
|
|
54
|
+
it somewhere the whole company can read. That refusal is correct; do not work
|
|
55
|
+
around it by relabelling the finding as something else.
|
|
56
|
+
|
|
57
|
+
## What is not yours
|
|
58
|
+
|
|
59
|
+
Spec drift and error-shape inconsistency are CONDUIT's unless the leak has a
|
|
60
|
+
security consequence. Schema constraints are VAULT's. A missing index is nobody's
|
|
61
|
+
security problem. When a finding is genuinely both, report the security
|
|
62
|
+
consequence and say which other surface it also touches.
|
|
@@ -11,7 +11,7 @@ one repository's directory layout or one app's seeded users works exactly once.
|
|
|
11
11
|
|
|
12
12
|
from __future__ import annotations
|
|
13
13
|
|
|
14
|
-
from qaas.config import SystemConfig
|
|
14
|
+
from qaas.config import AgentSpec, SystemConfig
|
|
15
15
|
from qaas.envelope import DefectEnvelope
|
|
16
16
|
from qaas.target import TargetProfile
|
|
17
17
|
|
|
@@ -218,6 +218,49 @@ fails, misleads, blocks or excludes someone. Do not report what you would have
|
|
|
218
218
|
designed differently."""
|
|
219
219
|
|
|
220
220
|
|
|
221
|
+
def discovery(config: SystemConfig, mode: str, spec: "AgentSpec") -> str:
|
|
222
|
+
"""The task for a discovery agent with no hand-written builder.
|
|
223
|
+
|
|
224
|
+
The architecture's claim is that adding an agent needs a prompt file and a
|
|
225
|
+
YAML file and no Python. That was not true: `_phase_discover` dispatched
|
|
226
|
+
from a hardcoded dict of builders, so a new discovery agent was silently
|
|
227
|
+
skipped with `no task builder` -- it validated, it assembled, it appeared in
|
|
228
|
+
`--dry-run`, and then it did nothing. VAULT and WARDEN were added exactly
|
|
229
|
+
that way and this is the bug they found.
|
|
230
|
+
|
|
231
|
+
What an agent should be told is: which application, what it can reach, and
|
|
232
|
+
what its own prompt says its domain is. Everything specific to a domain
|
|
233
|
+
belongs in that agent's prompt, not here -- CONDUIT and SURFACE keep their
|
|
234
|
+
bespoke builders because they name tools (`diff_openapi`, the browser) that
|
|
235
|
+
only they have.
|
|
236
|
+
"""
|
|
237
|
+
p = _profile(config)
|
|
238
|
+
reach = (
|
|
239
|
+
"The application is reachable, so prove what you report: observe the "
|
|
240
|
+
"behaviour and capture the evidence. A finding you have not observed is a "
|
|
241
|
+
"hypothesis, and its confidence should say so."
|
|
242
|
+
if p.environment.is_reachable
|
|
243
|
+
else "There is no reachable instance, so every finding is a reading of the "
|
|
244
|
+
"code. Quote the lines that support it and keep your confidence honest "
|
|
245
|
+
"about not having observed the behaviour."
|
|
246
|
+
)
|
|
247
|
+
return f"""Audit {p.name} for defects in your domain.
|
|
248
|
+
|
|
249
|
+
Layout — {p.layout.described()}
|
|
250
|
+
|
|
251
|
+
Your own instructions define what your domain is and what counts as evidence in
|
|
252
|
+
it. Work within it and leave the other surfaces to the agents that own them.
|
|
253
|
+
|
|
254
|
+
{reach}
|
|
255
|
+
|
|
256
|
+
Emit one envelope per distinct defect with `emit_envelope`. Finding nothing is a
|
|
257
|
+
valid outcome; inventing something to report is not. Deduplicate against
|
|
258
|
+
`search_similar` before you emit, so a defect this system already knows about
|
|
259
|
+
comes back as an occurrence rather than a new finding.
|
|
260
|
+
|
|
261
|
+
Mode: {mode}."""
|
|
262
|
+
|
|
263
|
+
|
|
221
264
|
def forge(envelope: DefectEnvelope, config: SystemConfig, flake_runs: int) -> str:
|
|
222
265
|
p = _profile(config)
|
|
223
266
|
evidence = "\n".join(f" - {e.type.value}: {e.uri} {e.note}".rstrip() for e in envelope.evidence)
|
|
@@ -543,3 +543,21 @@ async def test_a_resumed_run_keeps_the_budget_it_already_spent(cfg, tmp_path, fa
|
|
|
543
543
|
|
|
544
544
|
assert report.stopped_early, "the resumed run ignored what the run had already spent"
|
|
545
545
|
assert "spend cap" in report.stopped_early
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
async def test_a_new_discovery_agent_runs_without_a_hand_written_task(cfg, tmp_path, fake_agents):
|
|
549
|
+
"""The architecture's central claim, finally tested.
|
|
550
|
+
|
|
551
|
+
`_phase_discover` used to dispatch from a closed dict of task builders, so a
|
|
552
|
+
discovery agent that was not in it got `skipped: no task builder` -- it
|
|
553
|
+
passed `qaas validate`, assembled correctly, appeared in `--dry-run`, and
|
|
554
|
+
then silently did nothing. VAULT and WARDEN were added as a prompt plus a
|
|
555
|
+
YAML each and found exactly that.
|
|
556
|
+
|
|
557
|
+
A silent skip is the worst shape this failure could take, which is why this
|
|
558
|
+
test asserts the agent RAN rather than asserting the config loaded.
|
|
559
|
+
"""
|
|
560
|
+
calls, behaviour = fake_agents
|
|
561
|
+
report = await make_conductor(cfg, tmp_path).run("nightly")
|
|
562
|
+
ran = {name for name, _ in calls}
|
|
563
|
+
assert {"VAULT", "WARDEN"} <= ran, f"a config-only agent was skipped: {sorted(ran)}"
|
|
@@ -15,7 +15,12 @@ PROMPTS = REPO / "src" / "qaas" / "prompts"
|
|
|
15
15
|
|
|
16
16
|
PHASE_1 = {"CARTOGRAPHER", "CONDUIT", "SURFACE", "FORGE", "CLERK", "PROOF"}
|
|
17
17
|
PHASE_3 = {"MENDER", "ARBITER"}
|
|
18
|
-
|
|
18
|
+
#: Added later, and the point of them is how they were added: a prompt file and a
|
|
19
|
+
#: YAML file each, with no change to conductor, runner, registry or guardrails.
|
|
20
|
+
#: That was the architecture's central claim and it went untested until someone
|
|
21
|
+
#: actually tried it.
|
|
22
|
+
PHASE_2 = {"VAULT", "WARDEN"}
|
|
23
|
+
ROSTER = PHASE_1 | PHASE_3 | PHASE_2
|
|
19
24
|
|
|
20
25
|
|
|
21
26
|
@pytest.fixture(scope="module")
|
|
@@ -19,6 +19,9 @@ import pytest
|
|
|
19
19
|
from support import CONFIG_SEARCH, PACKAGED_CONFIG, PACKAGED_PROMPTS, PACKAGED_SKILLS
|
|
20
20
|
|
|
21
21
|
from qaas.config import load_config
|
|
22
|
+
|
|
23
|
+
#: However many ship; the point is that shadowing one does not lose the rest.
|
|
24
|
+
PACKAGED_AGENTS = sorted((PACKAGED_CONFIG / "agents").glob("*.yaml"))
|
|
22
25
|
from qaas.paths import Workspace, find_project, package_root
|
|
23
26
|
|
|
24
27
|
|
|
@@ -115,7 +118,7 @@ def test_one_agent_file_can_be_shadowed_without_forking_the_rest(tmp_path):
|
|
|
115
118
|
|
|
116
119
|
cfg = load_config(search=(tmp_path / "config", *CONFIG_SEARCH))
|
|
117
120
|
assert cfg.agents["MENDER"].max_budget_usd == 9.0, "the override did not win"
|
|
118
|
-
assert len(cfg.agents) ==
|
|
121
|
+
assert len(cfg.agents) == len(PACKAGED_AGENTS), "shadowing one agent must not drop the others"
|
|
119
122
|
assert cfg.agents["CONDUIT"].max_budget_usd == 3.0, "an untouched agent changed"
|
|
120
123
|
|
|
121
124
|
|
|
@@ -192,14 +192,24 @@ codebase is strict about it:
|
|
|
192
192
|
| CARTOGRAPHER | map | services, routes, schema, ownership → `system-map.json` |
|
|
193
193
|
| CONDUIT | discovery | API contract drift; ships a failing contract test |
|
|
194
194
|
| SURFACE | discovery | drives the UI through real journeys |
|
|
195
|
+
| VAULT | discovery | schema constraints the code assumes and the database does not enforce |
|
|
196
|
+
| WARDEN | discovery | missing authorization, secrets, vulnerable dependencies, leaked internals |
|
|
195
197
|
| FORGE | triage | reproduces, minimises, measures flake, commits a failing test |
|
|
196
198
|
| CLERK | triage | dedupes, scores severity, routes, files — the only tracker writer |
|
|
197
199
|
| MENDER | remediation | the minimal fix, on a `fix/*` branch |
|
|
198
200
|
| ARBITER | remediation | adversarial review: APPROVE / REQUEST_CHANGES / ESCALATE |
|
|
199
201
|
| PROOF | verify | re-runs the original test → VERIFIED / NOT_FIXED / REGRESSED |
|
|
200
202
|
|
|
201
|
-
The design names 16.
|
|
202
|
-
|
|
203
|
+
The design names 16. Ten are built; CONDUCTOR is the state machine rather than an
|
|
204
|
+
agent; five further discovery specialists are designed and not written.
|
|
205
|
+
|
|
206
|
+
VAULT and WARDEN are worth noting for *how* they were added: a prompt file and a
|
|
207
|
+
YAML file each, with no Python. That is the architecture's central claim, and it
|
|
208
|
+
was not quite true until they tested it -- `_phase_discover` dispatched from a
|
|
209
|
+
closed dict of task builders, so a new discovery agent validated, assembled,
|
|
210
|
+
appeared in `--dry-run`, and was then silently skipped with `no task builder`.
|
|
211
|
+
There is a generic discovery task now, and a test that asserts a config-only
|
|
212
|
+
agent actually *runs*.
|
|
203
213
|
|
|
204
214
|
---
|
|
205
215
|
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
:root{--ink: #202124;--ink-muted: #5f6368;--surface: #ffffff;--canvas: #f8f9fa;--line: #dadce0;--accent: #1a56b0;--accent-dark: #14428a;--danger: #b3261e;--radius: 6px;--font: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif}*{box-sizing:border-box}html,body{margin:0;padding:0}body{background:var(--canvas);color:var(--ink);font-family:var(--font);font-size:15px;line-height:1.5}#root{min-height:100vh}h1,h2{margin:0;font-weight:600}.app-shell{min-height:100vh;display:flex;flex-direction:column}.app-header{background:var(--surface);border-bottom:1px solid var(--line)}.app-header-inner{max-width:1040px;margin:0 auto;padding:12px 20px;display:flex;align-items:center;gap:24px}.app-brand{font-weight:700;font-size:16px;color:var(--ink)}.app-nav{display:flex;gap:16px;flex:1}.nav-link{color:var(--accent);text-decoration:none;padding:4px 2px;border-bottom:2px solid transparent}.nav-link:hover{text-decoration:underline}.nav-link-active{color:var(--ink);border-bottom-color:var(--accent)}.app-account{display:flex;align-items:center;gap:12px}.app-account-email{color:var(--ink-muted);font-size:13px}.app-account-role{margin-left:8px;padding:1px 6px;border:1px solid var(--line);border-radius:10px;text-transform:uppercase;font-size:11px;letter-spacing:.04em}.app-main{flex:1;width:100%;max-width:1040px;margin:0 auto;padding:24px 20px 48px}.page{display:flex;flex-direction:column;gap:16px}.page-header{display:flex;align-items:center;gap:12px}.page-title{font-size:22px}.section-title{font-size:15px;margin-top:8px}.card{background:var(--surface);border:1px solid var(--line);border-radius:var(--radius);padding:20px;display:flex;flex-direction:column;gap:16px}.muted{color:var(--ink-muted)}.link{color:#15c}.empty-state{align-items:flex-start;gap:8px}.toolbar{display:flex;align-items:flex-end;gap:16px;flex-wrap:wrap}.field{display:flex;flex-direction:column;gap:4px}.field-inline{min-width:180px}.field-grow{flex:1;min-width:180px}.field-narrow{width:110px}.field-label{font-size:13px;font-weight:600;color:var(--ink)}.field-input,.search-input{font:inherit;color:var(--ink);background:var(--surface);border:1px solid var(--line);border-radius:var(--radius);padding:8px 10px;width:100%}.search-input{min-width:260px}.field-input:focus-visible,.search-input:focus-visible,.btn:focus-visible,.table-row-clickable:focus-visible,.nav-link:focus-visible,.link:focus-visible{outline:2px solid var(--accent);outline-offset:2px}::placeholder{color:var(--ink-muted);opacity:1}.item-row{display:flex;align-items:flex-end;gap:12px;flex-wrap:wrap;padding-bottom:12px;border-bottom:1px solid var(--line)}.form-actions{display:flex;gap:12px;align-items:center}.btn{font:inherit;font-weight:600;border-radius:var(--radius);border:1px solid transparent;padding:8px 16px;cursor:pointer;text-decoration:none;display:inline-block}.btn:disabled{cursor:not-allowed;opacity:.6}.btn-primary{background:var(--accent);color:#fff;border-color:var(--accent)}.btn-primary:hover:not(:disabled){background:var(--accent-dark);border-color:var(--accent-dark)}.btn-secondary{background:#f1f3f4;color:#9aa0a6;border-color:var(--line)}.btn-secondary:hover:not(:disabled){background:#e8eaed}.btn-danger{background:var(--danger);color:#fff;border-color:var(--danger)}.table{width:100%;border-collapse:collapse;background:var(--surface);border:1px solid var(--line);border-radius:var(--radius)}.table th,.table td{text-align:left;padding:10px 12px;border-bottom:1px solid var(--line)}.table th{background:var(--canvas);font-size:13px;letter-spacing:.02em}.table tbody tr:last-child td{border-bottom:none}.table-row-clickable{cursor:pointer}.table-row-clickable:hover{background:#f1f6ff}.badge{display:inline-block;padding:2px 8px;border-radius:10px;font-size:12px;font-weight:600;text-transform:capitalize;color:var(--ink);background:#f1f3f4}.badge-draft{background:#eceff1}.badge-placed{background:#e8f0fe}.badge-paid{background:#e6f4ea}.badge-refunded{background:#fef7e0}.badge-cancelled{background:#fce8e6}.alert{margin:0;padding:10px 12px;border-radius:var(--radius);font-size:14px}.alert-error{background:#fce8e6;color:var(--danger);border:1px solid #f3c9c5}.alert-success{background:#e6f4ea;color:#0b6b34;border:1px solid #b7e0c4}.summary{margin:0;display:flex;flex-direction:column;gap:8px}.summary-row{display:flex;gap:12px}.summary-row dt{width:120px;color:var(--ink-muted);font-size:13px}.summary-row dd{margin:0}.total-line{margin:0;text-align:right;font-size:15px}.login-page{min-height:100vh;display:flex;align-items:center;justify-content:center;padding:24px}.login-card{width:100%;max-width:380px}.login-title{font-size:20px}.login-subtitle{margin:0;color:var(--ink-muted);font-size:14px}
|