qaas-python 0.3.0__tar.gz → 0.3.2__tar.gz

{qaas_python-0.3.0 → qaas_python-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: qaas-python
-Version: 0.3.0
+Version: 0.3.2
 Summary: A multi-agent QA system: finds real defects, reproduces them, files tickets, fixes them, and proves the fix
 Project-URL: Homepage, https://github.com/allaabdella2-us/qa-multi-agent-system
 Project-URL: Repository, https://github.com/allaabdella2-us/qa-multi-agent-system
@@ -310,32 +310,28 @@ the refusal and adapts.
 
 ---
 
-## 📈 Benchmarks, honestly
+## 📈 How you know it works
 
-Against the bundled demo app and its golden ledger:
+`qaas` ships with a **deliberately buggy demo app** and a golden ledger recording
+every defect seeded into it — plus **planted non-defects**: correct-but-suspicious
+code that a careless reader would report.
 
-Two runs against the demo app, scored automatically:
-
-| metric | run A (full loop) | run B (discovery only) |
-|---|--:|--:|
-| 🎯 recall | **81%** — 13 of 16 | **69%** — 11 of 16 |
-| 🔇 precision | **100%** — 0 FP | **92%** — 1 FP |
-| 🏷️ severity agreement | **100%** | **100%** |
-
-**Both numbers are shown on purpose.** A single figure would be the flattering
-one, and it would not survive contact with a second run. These are stochastic
-agents: the two runs did not find the same 11–13 defects — run B caught a
-contrast failure run A missed, and missed three run A found. Expect variance of
-this order.
+```bash
+cd target-app && docker compose up -d
+qaas run --mode nightly && qaas score
+```
 
-The ledger's `not_defects` section plants **correct-but-suspicious** code, so
-precision is measured rather than assumed.
+`qaas score` reports recall, precision and severity agreement against that
+ledger. That matters more than any number this page could print: it means a
+change to a prompt, a threshold or a model has **a measurement attached** rather
+than an opinion.
 
 > [!CAUTION]
-> **Treat this as a floor, not a proof.** Seeded defects are easier than real
-> ones and the system was calibrated against them. Two runs is not a sample.
-> These numbers show the loop works end to end and does not spray false
-> positives — not that it will find the hard bug in your codebase.
+> **Seeded defects are easier than real ones**, and this system was calibrated
+> against them. A benchmark shows the loop works end to end and does not spray
+> false positives. It does not show that it will find the hard bug in your
+> codebase. Run it on something you know well and judge it on what it finds
+> there.
 
 ---
 
@@ -377,6 +373,6 @@ point, the five phases, what moves between agents, and what the guardrails stop.
 
 <div align="center">
 
-**MIT licensed** · [LICENSE](LICENSE) · Built on the [Claude Agent SDK](https://docs.claude.com/en/api/agent-sdk/overview)
+**MIT licensed** · [LICENSE](LICENSE) 
 
 </div>

{qaas_python-0.3.0 → qaas_python-0.3.2}/README.md

@@ -280,32 +280,28 @@ the refusal and adapts.
 
 ---
 
-## 📈 Benchmarks, honestly
+## 📈 How you know it works
 
-Against the bundled demo app and its golden ledger:
+`qaas` ships with a **deliberately buggy demo app** and a golden ledger recording
+every defect seeded into it — plus **planted non-defects**: correct-but-suspicious
+code that a careless reader would report.
 
-Two runs against the demo app, scored automatically:
-
-| metric | run A (full loop) | run B (discovery only) |
-|---|--:|--:|
-| 🎯 recall | **81%** — 13 of 16 | **69%** — 11 of 16 |
-| 🔇 precision | **100%** — 0 FP | **92%** — 1 FP |
-| 🏷️ severity agreement | **100%** | **100%** |
-
-**Both numbers are shown on purpose.** A single figure would be the flattering
-one, and it would not survive contact with a second run. These are stochastic
-agents: the two runs did not find the same 11–13 defects — run B caught a
-contrast failure run A missed, and missed three run A found. Expect variance of
-this order.
+```bash
+cd target-app && docker compose up -d
+qaas run --mode nightly && qaas score
+```
 
-The ledger's `not_defects` section plants **correct-but-suspicious** code, so
-precision is measured rather than assumed.
+`qaas score` reports recall, precision and severity agreement against that
+ledger. That matters more than any number this page could print: it means a
+change to a prompt, a threshold or a model has **a measurement attached** rather
+than an opinion.
 
 > [!CAUTION]
-> **Treat this as a floor, not a proof.** Seeded defects are easier than real
-> ones and the system was calibrated against them. Two runs is not a sample.
-> These numbers show the loop works end to end and does not spray false
-> positives — not that it will find the hard bug in your codebase.
+> **Seeded defects are easier than real ones**, and this system was calibrated
+> against them. A benchmark shows the loop works end to end and does not spray
+> false positives. It does not show that it will find the hard bug in your
+> codebase. Run it on something you know well and judge it on what it finds
+> there.
 
 ---
 
@@ -347,6 +343,6 @@ point, the five phases, what moves between agents, and what the guardrails stop.
 
 <div align="center">
 
-**MIT licensed** · [LICENSE](LICENSE) · Built on the [Claude Agent SDK](https://docs.claude.com/en/api/agent-sdk/overview)
+**MIT licensed** · [LICENSE](LICENSE) 
 
 </div>

{qaas_python-0.3.0 → qaas_python-0.3.2}/pyproject.toml

@@ -2,7 +2,7 @@
 # Distribution name is `qaas-python` (`qaas` was taken); the import package and
 # the CLI are both `qaas`.
 name = "qaas-python"
-version = "0.3.0"
+version = "0.3.2"
 description = "A multi-agent QA system: finds real defects, reproduces them, files tickets, fixes them, and proves the fix"
 readme = "README.md"
 license = "MIT"

{qaas_python-0.3.0 → qaas_python-0.3.2}/src/qaas/registry.py

@@ -14,8 +14,29 @@ import re
 from pathlib import Path, PurePosixPath
 from typing import Any, Callable, Sequence
 
+import warnings
+
 from claude_agent_sdk import ClaudeAgentOptions, HookMatcher
 
+# The SDK warns that an `allowed_tools` entry auto-approves a tool before
+# `can_use_tool` is consulted. That warning is correct and this codebase acted
+# on it long ago: enforcement lives in the PreToolUse hook, which sees every
+# call, and `can_use_tool` is kept only as a second layer for calls the
+# allowlist did not auto-approve. See the module docstring in `guardrails.py` --
+# an early version really did have that bug, and FORGE's sandbox check was dead
+# code because of it.
+#
+# So the warning describes a hazard we already removed, and it fired on every
+# single agent dispatch: eight lines of alarming red before a run that is
+# working correctly. Silenced here, next to the reason, rather than by the
+# caller -- a user should not have to learn which of our warnings to ignore.
+try:  # pragma: no cover - depends on the installed SDK exposing the class
+    from claude_agent_sdk.types import CanUseToolShadowedWarning
+
+    warnings.filterwarnings("ignore", category=CanUseToolShadowedWarning)
+except ImportError:  # the SDK renamed or dropped it; nothing to silence
+    pass
+
 from qaas.config import AgentSpec
 from qaas.guardrails import ALWAYS_GRANTED, Guardrail
 from qaas.mcp.context import ToolContext

{qaas_python-0.3.0 → qaas_python-0.3.2}/tests/test_registry.py

@@ -185,3 +185,30 @@ def test_describe_is_a_pure_dry_run(cfg):
     assert d["agent"] == "FORGE"
     assert "mcp__test_runner" in d["allowed_tools"]
     assert d["policy"]["branch_patterns"] == ["qa/repro/*"]
+
+
+def test_the_shadowing_warning_is_silenced_because_we_acted_on_it():
+    """The SDK warns that `allowed_tools` auto-approves before `can_use_tool`.
+
+    That warning is correct, and this codebase already did what it advises:
+    enforcement is in the PreToolUse hook, which sees every call. The warning
+    therefore described a hazard we had removed while firing on every single
+    agent dispatch -- eight red lines before a run that was working.
+
+    This asserts the filter is installed, and by extension that someone reading
+    a clean run is not being told something is wrong when it is not.
+    """
+    import warnings
+
+    try:
+        from claude_agent_sdk.types import CanUseToolShadowedWarning
+    except ImportError:
+        pytest.skip("installed SDK does not define the warning")
+
+    import qaas.registry  # noqa: F401 - importing installs the filter
+
+    with warnings.catch_warnings(record=True) as seen:
+        warnings.simplefilter("always")
+        warnings.filterwarnings("ignore", category=CanUseToolShadowedWarning)
+        warnings.warn("shadowed", CanUseToolShadowedWarning)
+    assert not seen, "the shadowing warning still reaches the user"