argus-testing 0.7.1__tar.gz → 0.9.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (183) hide show
  1. {argus_testing-0.7.1 → argus_testing-0.9.0}/.gitignore +3 -0
  2. {argus_testing-0.7.1 → argus_testing-0.9.0}/AGENTS.md +30 -9
  3. {argus_testing-0.7.1 → argus_testing-0.9.0}/CHANGELOG.md +41 -0
  4. {argus_testing-0.7.1 → argus_testing-0.9.0}/CONTRIBUTING.md +18 -0
  5. {argus_testing-0.7.1 → argus_testing-0.9.0}/PKG-INFO +146 -40
  6. {argus_testing-0.7.1 → argus_testing-0.9.0}/PRIVACY.md +5 -0
  7. {argus_testing-0.7.1 → argus_testing-0.9.0}/README.md +144 -38
  8. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/assets/dashboard.html +1 -1
  9. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/assets/dashboard.js +4 -0
  10. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/monitoring.py +8 -0
  11. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/actions.py +32 -13
  12. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/coverage.py +3 -3
  13. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/expectations.py +70 -11
  14. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/findings.py +55 -11
  15. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/finish.py +19 -2
  16. argus_testing-0.9.0/argus/application/guidance.py +105 -0
  17. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/journeys.py +45 -9
  18. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/lifecycle.py +14 -5
  19. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/manager.py +64 -28
  20. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/navigation.py +11 -3
  21. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/observation.py +64 -41
  22. argus_testing-0.9.0/argus/application/payloads.py +49 -0
  23. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/probes.py +19 -3
  24. argus_testing-0.9.0/argus/application/progress.py +72 -0
  25. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/session.py +3 -0
  26. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/browser.py +10 -0
  27. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/domain/contracts.py +11 -0
  28. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/entrypoints/mcp/catalog.py +22 -0
  29. argus_testing-0.9.0/argus/entrypoints/mcp/overhead.py +40 -0
  30. argus_testing-0.9.0/argus/entrypoints/mcp/results.py +98 -0
  31. argus_testing-0.9.0/argus/entrypoints/mcp/server.py +134 -0
  32. argus_testing-0.9.0/argus/entrypoints/mcp/snapshots.py +77 -0
  33. argus_testing-0.9.0/argus/evaluation/__init__.py +1 -0
  34. argus_testing-0.9.0/argus/evaluation/__main__.py +68 -0
  35. argus_testing-0.9.0/argus/evaluation/adjudication.py +114 -0
  36. argus_testing-0.9.0/argus/evaluation/agent_fixture.py +156 -0
  37. argus_testing-0.9.0/argus/evaluation/browser_baseline.py +89 -0
  38. argus_testing-0.9.0/argus/evaluation/codex_runner.py +344 -0
  39. argus_testing-0.9.0/argus/evaluation/fixture.py +59 -0
  40. argus_testing-0.9.0/argus/evaluation/gateway.py +111 -0
  41. argus_testing-0.9.0/argus/evaluation/models.py +117 -0
  42. argus_testing-0.9.0/argus/evaluation/overhead.py +121 -0
  43. argus_testing-0.9.0/argus/evaluation/scoring.py +94 -0
  44. argus_testing-0.9.0/argus/evaluation/smoke.py +153 -0
  45. argus_testing-0.9.0/argus/install_check.py +68 -0
  46. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/mcp_server.py +7 -2
  47. argus_testing-0.9.0/argus/measurement.py +61 -0
  48. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/reporter.py +2 -0
  49. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/settings.py +5 -0
  50. argus_testing-0.9.0/docs/agent-protocol.md +176 -0
  51. {argus_testing-0.7.1 → argus_testing-0.9.0}/docs/architecture.md +45 -0
  52. {argus_testing-0.7.1 → argus_testing-0.9.0}/docs/decisions.md +66 -0
  53. argus_testing-0.9.0/docs/evaluation-0.8.md +122 -0
  54. argus_testing-0.9.0/docs/evaluation.md +208 -0
  55. argus_testing-0.9.0/examples/agent-evaluation-optimized-evidence.json +4008 -0
  56. argus_testing-0.9.0/examples/agent-evaluation-optimized.json +939 -0
  57. argus_testing-0.9.0/examples/agent-evaluation-pilot-evidence.json +2349 -0
  58. argus_testing-0.9.0/examples/agent-evaluation-pilot.json +409 -0
  59. argus_testing-0.9.0/examples/response-overhead.json +170 -0
  60. {argus_testing-0.7.1 → argus_testing-0.9.0}/llms-install.md +22 -0
  61. {argus_testing-0.7.1 → argus_testing-0.9.0}/pyproject.toml +3 -2
  62. {argus_testing-0.7.1 → argus_testing-0.9.0}/scripts/check_distributions.py +3 -0
  63. {argus_testing-0.7.1 → argus_testing-0.9.0}/server.json +4 -4
  64. argus_testing-0.9.0/tests/test_adjudication.py +104 -0
  65. argus_testing-0.9.0/tests/test_agent_results.py +155 -0
  66. argus_testing-0.9.0/tests/test_agent_workflow.py +172 -0
  67. argus_testing-0.9.0/tests/test_codex_evaluation.py +154 -0
  68. argus_testing-0.9.0/tests/test_evaluation.py +193 -0
  69. {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_expectation_oracle.py +4 -0
  70. {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_mcp_contract.py +3 -1
  71. argus_testing-0.9.0/tests/test_monitoring.py +198 -0
  72. {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_release_metadata.py +16 -2
  73. argus_testing-0.9.0/tests/test_response_efficiency.py +235 -0
  74. argus_testing-0.9.0/tests/test_stdio_protocol.py +132 -0
  75. argus_testing-0.7.1/argus/entrypoints/mcp/server.py +0 -74
  76. argus_testing-0.7.1/tests/test_monitoring.py +0 -111
  77. argus_testing-0.7.1/tests/test_stdio_protocol.py +0 -54
  78. {argus_testing-0.7.1 → argus_testing-0.9.0}/LICENSE +0 -0
  79. {argus_testing-0.7.1 → argus_testing-0.9.0}/SECURITY.md +0 -0
  80. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/__init__.py +0 -0
  81. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/_compat.py +0 -0
  82. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/access.py +0 -0
  83. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/__init__.py +0 -0
  84. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/assets/__init__.py +0 -0
  85. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/assets/dashboard.css +0 -0
  86. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/journal.py +0 -0
  87. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/logging.py +0 -0
  88. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/reporting/__init__.py +0 -0
  89. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/reporting/common.py +0 -0
  90. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/reporting/html.py +0 -0
  91. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/reporting/json.py +0 -0
  92. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/reporting/junit.py +0 -0
  93. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/reporting/privacy.py +0 -0
  94. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/reporting/sarif.py +0 -0
  95. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/scripts/__init__.py +0 -0
  96. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/scripts/check_layout_js.js +0 -0
  97. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/scripts/extract_elements_js.js +0 -0
  98. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/scripts/extract_page_content_js.js +0 -0
  99. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/scripts/extract_performance_js.js +0 -0
  100. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/scripts/find_visible_element_js.js +0 -0
  101. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/scripts/inspect_element_js.js +0 -0
  102. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/__init__.py +0 -0
  103. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/capsules.py +0 -0
  104. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/capture.py +0 -0
  105. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/context.py +0 -0
  106. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/events.py +0 -0
  107. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/evidence.py +0 -0
  108. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/inspection.py +0 -0
  109. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/network_storage.py +0 -0
  110. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/operations.py +0 -0
  111. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/regression.py +0 -0
  112. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/screen.py +0 -0
  113. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/screenshots.py +0 -0
  114. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/telemetry.py +0 -0
  115. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/text.py +0 -0
  116. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/verification.py +0 -0
  117. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/verifiers.py +0 -0
  118. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/bench/__init__.py +0 -0
  119. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/bench/__main__.py +0 -0
  120. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/bench/agent_runner.py +0 -0
  121. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/bench/runner.py +0 -0
  122. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/bench/scenarios_buggytasks.py +0 -0
  123. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/bench/scenarios_darkshop.py +0 -0
  124. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/bootstrap.py +0 -0
  125. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/cleanup.py +0 -0
  126. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/cli.py +0 -0
  127. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/config.py +0 -0
  128. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/detector.py +0 -0
  129. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/differ.py +0 -0
  130. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/domain/__init__.py +0 -0
  131. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/domain/detector.py +0 -0
  132. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/domain/models.py +0 -0
  133. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/domain/replay.py +0 -0
  134. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/domain/verification.py +0 -0
  135. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/entrypoints/__init__.py +0 -0
  136. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/entrypoints/mcp/__init__.py +0 -0
  137. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/explorer.py +0 -0
  138. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/models.py +0 -0
  139. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/planner.py +0 -0
  140. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/ports.py +0 -0
  141. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/resolver.py +0 -0
  142. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/runtime.py +0 -0
  143. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/screen/__init__.py +0 -0
  144. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/screen/backend.py +0 -0
  145. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/screen/permissions.py +0 -0
  146. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/screen/safety.py +0 -0
  147. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/screen/validate.py +0 -0
  148. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/security.py +0 -0
  149. {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/storage.py +0 -0
  150. {argus_testing-0.7.1 → argus_testing-0.9.0}/docs/implementation-0.7.md +0 -0
  151. {argus_testing-0.7.1 → argus_testing-0.9.0}/examples/focus.yaml +0 -0
  152. {argus_testing-0.7.1 → argus_testing-0.9.0}/examples/mcp-config.json +0 -0
  153. {argus_testing-0.7.1 → argus_testing-0.9.0}/glama.json +0 -0
  154. {argus_testing-0.7.1 → argus_testing-0.9.0}/human-eye-fixture/app.py +0 -0
  155. {argus_testing-0.7.1 → argus_testing-0.9.0}/test-site/app.py +0 -0
  156. {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/__init__.py +0 -0
  157. {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/conftest.py +0 -0
  158. {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_action_targeting.py +0 -0
  159. {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_architecture_runtime.py +0 -0
  160. {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_bench_report.py +0 -0
  161. {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_browser_shadow.py +0 -0
  162. {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_cross_stack.py +0 -0
  163. {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_detector_console.py +0 -0
  164. {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_detector_network.py +0 -0
  165. {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_distribution_contents.py +0 -0
  166. {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_final_hardening.py +0 -0
  167. {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_journal_regression.py +0 -0
  168. {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_regression_cli.py +0 -0
  169. {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_replay_minimize.py +0 -0
  170. {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_replay_receipt.py +0 -0
  171. {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_replay_trace.py +0 -0
  172. {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_reporter_badge.py +0 -0
  173. {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_reproduction_receipt.py +0 -0
  174. {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_resolver.py +0 -0
  175. {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_runtime_limits.py +0 -0
  176. {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_screen_permissions.py +0 -0
  177. {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_screen_safety.py +0 -0
  178. {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_security_boundaries.py +0 -0
  179. {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_session_lifecycle.py +0 -0
  180. {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_state_capsules.py +0 -0
  181. {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_storage_cleanup.py +0 -0
  182. {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_substrate.py +0 -0
  183. {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_visual_inspection.py +0 -0
@@ -20,3 +20,6 @@ docs/*
20
20
  !docs/architecture.md
21
21
  !docs/decisions.md
22
22
  !docs/implementation-0.7.md
23
+ !docs/agent-protocol.md
24
+ !docs/evaluation.md
25
+ !docs/evaluation-0.8.md
@@ -25,7 +25,18 @@ argus-mcp --list-tools # show the selected profile and public tools
25
25
  argus-mcp --doctor # macOS Screen Recording + Accessibility check
26
26
  ```
27
27
 
28
- The default profile is `core`. Use `--tool-profile screen` for native
28
+ The default profile is `core`. For structured-result-aware hosts, use
29
+ `--tool-profile agent --response-mode compact` for 19 focused tools.
30
+
31
+ `--response-mode delta` is opt-in for snapshot-aware clients. Only observation
32
+ fields listed in the protocol may be inherited from an exact `base_id`; action
33
+ and verification results are current-call-only. If context or a base is missing,
34
+ call `observe` for full state. Never infer a passing check from unchanged fields.
35
+ Use `python -m argus.install_check` to verify installed versions and stdio without
36
+ a browser or model, and `python -m argus.evaluation.overhead` for deterministic
37
+ byte measurements. Bytes are not model tokens. Monitoring retains numeric
38
+ overhead only; do not add page content or arguments to metrics.
39
+ Use `--tool-profile screen` for native
29
40
  macOS review and `--tool-profile full` only when the broader network,
30
41
  storage, tabs, crawl, and coordinate surface is needed. The
31
42
  `ARGUS_TOOL_PROFILE` environment variable provides the same setting.
@@ -48,6 +59,16 @@ the user's broader task. The short version:
48
59
  Map → **Use it** (walk each goal end-to-end, carrying real state) →
49
60
  Hypothesize → Act → Observe → Verify → Record → Cover.
50
61
 
62
+ ## Structured results (0.8)
63
+
64
+ Read `structuredContent.data`
65
+ for operation-specific evidence where available; text and images remain for
66
+ compatibility. `status: completed` means the tool ran, not that the application
67
+ passed. Inspect expectation matches, finding verdicts and export warnings.
68
+ Progress notifications describe actual phases, not percentages or proof of
69
+ coverage. Missing live notifications do not mean execution stopped; the final
70
+ result includes captured stages. See [the protocol](docs/agent-protocol.md).
71
+
51
72
  ## Tools you'll use most
52
73
 
53
74
  ### Web mode
@@ -55,13 +76,13 @@ the user's broader task. The short version:
55
76
  | Tool | Purpose |
56
77
  |------|---------|
57
78
  | `start_session(url, review_mode=..., goals=[...], constraints=[...], time_budget_minutes=...)` | Launch Playwright, establish the review contract, and return the one-time QA protocol plus initial observation. The budget is advisory; `0` means none. |
58
- | `observe()` | URL + interactive elements (description-keyed) + visible feedback + counts + ARIA + viewport state. Read this first, after every action. |
59
- | `coverage_update(goal, status, evidence)` | Mark a goal `in_progress` before its journey to open a precise evidence window. `exercised` and `blocked` require an explanation and automatically link URLs, value-redacted actions, screenshots, checks, and findings; Argus never guesses semantic completion. |
79
+ | `observe()` | URL, targets, feedback, counts, coverage and next-check hints. Reuse post-action observations; call again only for missing or stale evidence. |
80
+ | `coverage_update(goal, status, evidence)` | Accepts one goal or a list of related goals. Mark `in_progress` before the shared journey. `exercised` and `blocked` require observed evidence and link URLs, value-redacted actions, screenshots, checks and findings. Invalid batches make no changes. |
60
81
  | `click_what(description)` | Click the element matching `description`. Returns the top candidates if ambiguous — rephrase rather than guess. |
61
82
  | `type_into(description, text)` / `select_into(description, value)` | Inputs and dropdowns by description. |
62
83
  | `test_action(target, expect=...)` | Click + before/after diff in one call. Pass `expect` to PREDICT the outcome ({"count":{"label":"tasks","delta":1}}, {"gains":"Buy milk"}, {"removes":...}, {"text_present":...}, {"toast":...}, {"url_changed":true}) and Argus reports MATCH / SURPRISE — a surprise is a bug lead. Also shows CROSS-STACK: which requests the click fired (methods/statuses) and a CHECK nudge when a message appeared without a matching write. |
63
- | `verify_persistence(expect, target_text, after_url)` | Forces a fresh GET; reports whether `target_text` is `present` / `absent`. The "Saved!" toast is not proof — this is. |
64
- | `capsule_save(name, liveness_marker)` / `capsule_restore(name)` | Snapshot the logged-in/seeded state (cookies+storage) after minting it through the UI, then restore it later (with a mandatory live/stale re-check). Restore is a CLEAN replace, so save→branch A→restore→branch B runs two journeys from a byte-identical state for differential testing. |
84
+ | `verify_persistence(expect, target_text, after_url)` | Forces a fresh GET; returns target presence and the page observation. Check persistence once, repeat only to resolve uncertainty. A matching symptom is not proof of a semantic defect. |
85
+ | `capsule_save(name, liveness_marker)` / `capsule_restore(name)` | Snapshot cookies and page storage after establishing state through the UI. Restore replaces the captured client state and checks a saved liveness marker when provided; without one, validity remains unverified. It does not reset backend data or guarantee identical application state across journeys. |
65
86
  | `regression_check()` | Re-test the findings journaled in prior runs against the CURRENT build: STILL-PRESENT / NO-LONGER-REPRODUCES / INCONCLUSIVE. "Did my fix land, did anything come back?" |
66
87
  | `inspect_element(description)` | Computed styles + ARIA + outerHTML + truncation flag for interactive or visible non-interactive content. |
67
88
  | `check_layout()` | Bounded viewport signals for horizontal overflow, clipped text, small targets, and fixed/sticky overlays. |
@@ -91,16 +112,16 @@ the user's broader task. The short version:
91
112
  ```
92
113
  start_session(url, goals=[...], constraints=[...], time_budget_minutes=15)
93
114
  # initial result already MAPS the page
94
- coverage_update(goal, "in_progress") # opens this goal's evidence window
115
+ coverage_update([goal_a, goal_b], "in_progress")
95
116
  # USE IT — pick a real goal, walk it
96
117
  # end-to-end, carrying state across pages
97
118
  # HYPOTHESIZE — what could go wrong
98
119
  click_what(...) / type_into(...) / # ACT — one probe per call
99
120
  test_form({...})
100
- observe() # OBSERVE — what changed?
121
+ # read returned observation; observe only if needed
101
122
  verify_persistence(...) # VERIFY (delete / save / submit / toggle)
102
123
  record_bug(..., verify={...}) # RECORD — verify clause attaches a receipt
103
- coverage_update(goal, "exercised", # COVER — preserve the observed evidence
124
+ coverage_update([goal_a, goal_b], "exercised",
104
125
  evidence)
105
126
  ... repeat ...
106
127
  end_session() # reports completed + unfinished coverage
@@ -157,7 +178,7 @@ Don't record speculation, polish nits, or static a11y / SEO that
157
178
  - `argus-mcp --version` — what version your MCP host actually loaded.
158
179
  - `argus-mcp --doctor` — macOS permission probes for screen mode.
159
180
  - `argus-mcp --list-tools` — selected tool profile and exported names.
160
- - `argus-mcp --tool-profile core|screen|full` — control discovery size.
181
+ - `argus-mcp --tool-profile agent|core|screen|full` — control discovery size.
161
182
  - `argus-mcp --unsafe` — turn on `eval_js` (off by default).
162
183
  - `argus-mcp --monitor-port 8765` — enable the read-only loopback dashboard,
163
184
  health/readiness endpoints, SSE events and metrics; use `0` for a free port.
@@ -1,5 +1,46 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.9.0 - 2026-10-25
4
+
5
+ - Report the Argus product version in the MCP handshake instead of the SDK
6
+ version. Add a no-browser installed-package stdio check and run it against
7
+ fresh release wheels outside the source checkout.
8
+ - Remove redundant output-schema display titles without changing validation
9
+ constraints or tool display titles.
10
+ - Add opt-in delta observations with explicit base IDs, field removals, full
11
+ resynchronization, lifecycle invalidation and bounded redacted retention.
12
+ Preserve errors, current-call verdicts, image blocks and compatibility defaults.
13
+ - Measure response bytes and instrumented browser waits, observation, screenshots,
14
+ report export and encoding. Expose numeric totals through loopback status and
15
+ Prometheus; keep page content and input values out of monitoring.
16
+ - Add reproducible, lossless encoding comparisons with navigation controls and
17
+ gateway discovery/response measurements. Do not equate byte savings with model
18
+ token savings or claim a new model effectiveness evaluation.
19
+
20
+ ## 0.8.0 - 2026-10-25
21
+
22
+ - Add an optional 19-tool agent profile with concise discovery descriptions,
23
+ compact structured responses, post-action/check observations and atomic batch
24
+ coverage updates. Keep core as the backward-compatible default.
25
+ - Add bounded next-check guidance, persistence reminders and stalled-exploration
26
+ signals. Do not automatically mark goals complete or findings confirmed.
27
+ - Treat missing counts and malformed expectations as unknown, not guessed
28
+ matches. Preserve partial form outcomes and stable finding references.
29
+ - Show real testing phases, goal/check counts and separate finding outcomes in
30
+ the read-only loopback dashboard, without exposing goal text or inputs.
31
+ - Add opt-in, budget-bounded Codex subscription collection, synthetic browser
32
+ fixtures, conventional-browser controls and independent evidence adjudication.
33
+ Model calls are never part of import, normal MCP usage or CI.
34
+
35
+ - Add versioned structured MCP results alongside compatible text/image content,
36
+ including structured evidence for core observation, coverage, verification, findings
37
+ and report flows, with explicit runtime error codes and recovery guidance.
38
+ - Emit bounded, request-local phase progress over MCP and local monitoring;
39
+ retain a final summary when a host does not display notifications.
40
+ - Add paired evaluation schemas and offline metrics with matched controls,
41
+ retained failures, duplicate/false-positive accounting and unknown usage.
42
+ A no-model browser/stdio smoke suite exercises buggy, fixed and clean controls.
43
+
3
44
  ## 0.7.1 - 2026-10-25
4
45
 
5
46
  - Require Python 3.11 or newer and raise Pillow/filelock/certifi minimum versions to
@@ -41,6 +41,14 @@ separate macOS manual check; Linux CI does not claim to test that integration.
41
41
  access `_page`, `_context`, `_locator` or other adapter private state.
42
42
  - Monitor additions: `adapters/monitoring.py` and its packaged `assets/`.
43
43
  Never add raw inputs, arbitrary URLs, page contents or credentials to events.
44
+ - MCP result schemas: `entrypoints/mcp/results.py`; application evidence uses
45
+ `ToolText` without importing MCP. Keep text/image compatibility and test real stdio.
46
+ - Progress: fixed phases in `application/progress.py`, emitted at actual work
47
+ boundaries through the runtime. Never accept page-supplied progress messages.
48
+ - Coverage hints: `application/guidance.py`; use observed facts and bounded output,
49
+ never model calls, automatic writes or inferred completion. Monitor only counts.
50
+ - Paired evaluation: `evaluation/`; keep model calls and external adjudication
51
+ outside the product runtime. Preserve both arms, failures and unknown usage.
44
52
  - Access rules: `access.py`; keep operator policy outside tool-controlled data.
45
53
  File tests must explicitly grant their temporary fixture directory.
46
54
  - Redaction: `security.py` and the shared reporting privacy wrapper. Test every
@@ -64,6 +72,16 @@ For the scripted benchmark, start `test-site/app.py` and
64
72
  a capability ceiling, not a real-agent success rate. Model-provider benchmarks
65
73
  consume subscription/API usage and must be explicitly authorized.
66
74
 
75
+ Run `python -m argus.evaluation --smoke` for the isolated, no-model paired
76
+ protocol check. See [evaluation methodology](docs/evaluation.md) before reporting
77
+ agent effectiveness. Protocol details are in [agent protocol](docs/agent-protocol.md).
78
+
79
+ The opt-in `evaluation.codex_runner` consumes subscription usage and requires
80
+ explicit authorization. Never add it to CI. Unit and stdio gateway tests do not
81
+ invoke a model. Test every fixture's buggy/fixed ground truth through real browser
82
+ interactions before collecting model trials. Keep raw collections and credentials
83
+ outside the public repository; only reviewed synthetic aggregate evidence belongs here.
84
+
67
85
  For release, synchronize `pyproject.toml` and both version fields in
68
86
  `server.json`, update `CHANGELOG.md`, build with `python -m build`, check with
69
87
  `twine check`, and smoke-test the wheel outside the source directory. JS and
@@ -1,7 +1,7 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: argus-testing
3
- Version: 0.7.1
4
- Summary: Autonomous QA MCP that tests apps like a real testing engineer and verifies every certified bug
3
+ Version: 0.9.0
4
+ Summary: Agent-first testing MCP with evidence-backed coverage, persistence checks and reproduction receipts
5
5
  Project-URL: Homepage, https://yichenwu.dev/projects/argus
6
6
  Project-URL: Documentation, https://github.com/chriswu727/argus#readme
7
7
  Project-URL: Repository, https://github.com/chriswu727/argus
@@ -56,9 +56,9 @@ Description-Content-Type: text/markdown
56
56
 
57
57
  # Argus
58
58
 
59
- **An MCP server that tests apps like a real testing engineer—exploring user journeys, discovering unscripted bugs, and proving each finding before reporting it.**
59
+ **A testing-engineer toolkit for AI agents—explore user journeys, check outcomes, and reproduce bug symptoms with evidence, through MCP.**
60
60
 
61
- Argus is an [MCP](https://modelcontextprotocol.io/) server. It adds evidence-first browser QA to Claude Code, Codex, Cursor, or any MCP host without taking over the host agent's identity or broader coding task. The agent explores, inspects, verifies persistence, and records reproducible bugs. Every certified finding is **independently re-confirmed from a clean page load** before it's reported.
61
+ Argus is an [MCP](https://modelcontextprotocol.io/) server that gives your coding agent a testing workflow: evidence-backed coverage, action-and-observation tools, persistence checks, independent reproduction receipts, and live progress. The host agent decides what to investigate; Argus helps it check what actually happened. Browser QA works across platforms, with a separate native macOS tool profile.
62
62
 
63
63
  [![PyPI](https://img.shields.io/pypi/v/argus-testing?color=1a7f37)](https://pypi.org/project/argus-testing/)
64
64
  [![Python](https://img.shields.io/pypi/pyversions/argus-testing)](https://pypi.org/project/argus-testing/)
@@ -67,13 +67,77 @@ Argus is an [MCP](https://modelcontextprotocol.io/) server. It adds evidence-fir
67
67
  [![Capability ceiling](https://img.shields.io/badge/bench-34%2F34-brightgreen)](#benchmarks)
68
68
  [![License: MIT](https://img.shields.io/badge/license-MIT-green)](LICENSE)
69
69
 
70
+ [![Argus Testing — Autonomous QA MCP server](https://mcprush.com/badge/argus-testing-autonomous-qa-mcp/card.svg)](https://mcprush.com/to/argus-testing-autonomous-qa-mcp?ref=badge)
71
+
70
72
  [Product page](https://yichenwu.dev/projects/argus) · [Quick start](#quick-start) · [Why Argus](#why-argus-is-different) · [Compared](#how-it-compares) · [Tools](#tool-surface) · [Benchmarks](#benchmarks)
71
73
 
72
74
  </div>
73
75
 
74
76
  ---
75
77
 
76
- ## The output
78
+ ## Agent-first testing
79
+
80
+ Argus adds a testing workflow to the agent you already use: inspect, act,
81
+ check persistence, independently reproduce a symptom, and link the evidence.
82
+ HTML reports are an optional way to review the evidence, not the agent interface.
83
+
84
+ Since 0.8.0, tool responses preserve their text and
85
+ images while adding a versioned `structuredContent` envelope and output schema.
86
+ Core actions, observation, verification, findings, coverage and report operations expose
87
+ machine-readable data; other tools currently expose status/error metadata with
88
+ their existing content. Long workflows emit real phase notifications to MCP
89
+ clients that request progress. Clients that do not display them still receive
90
+ the phase summary in the final result. See the [agent protocol](docs/agent-protocol.md).
91
+
92
+ - **Coverage guidance:** up to four evidence-based next checks, outstanding goals,
93
+ persistence reminders, input-boundary probes and stalled-exploration detection.
94
+ - **Action + observation:** click, fill, select, navigate and keyboard actions
95
+ return the current page evidence, reducing the need for a separate observation.
96
+ - **Conservative checks:** missing or malformed evidence stays unknown, not a
97
+ guessed pass or a fabricated bug. Findings link to stable session references.
98
+ - **Focused agent mode:** `argus-mcp --tool-profile agent --response-mode compact`
99
+ exposes 19 common testing tools with concise discovery descriptions, preserves structured evidence,
100
+ and avoids repeating migrated payloads as prose. Compatible mode remains the default.
101
+ - **Visible progress:** real phases, goal counts and check counts over MCP;
102
+ `--monitor-port 8765` adds the local live dashboard without exposing goal text.
103
+ - **Measured response overhead (0.9):** smaller output schemas, numeric payload
104
+ and component-timing counters, and opt-in `--response-mode delta` for clients
105
+ that retain observation snapshots. Missing bases require a full `observe`;
106
+ verification outcomes and image blocks are never replaced by cached verdicts.
107
+
108
+ Does installing Argus improve an agent? Measure it with the
109
+ [paired evaluation harness](docs/evaluation.md): the same model, task, fixture,
110
+ initial state and budget, with both arms and failures retained. The local scripted smoke
111
+ suite validates the harness; it is **not evidence of agent performance gains**.
112
+
113
+ The [0.8 Codex evaluation](docs/evaluation-0.8.md) includes ten matched pairs:
114
+ Argus found 6/6 defect opportunities versus 5/6 for the browser baseline, including
115
+ one post-hoc fixture defect. Both had zero observed false positives and equal
116
+ declared-goal coverage. Argus used more time and tokens. On four repeated pilot
117
+ cases, the targeted update reduced Argus's median calls from 17 to 9; this small
118
+ synthetic sample does not establish general superiority.
119
+
120
+ The no-model [response encoding benchmark](docs/evaluation.md#response-overhead)
121
+ compares identical synthetic evidence, including navigation invalidation. Delta
122
+ mode reduces aggregate result JSON by 47.09% versus compact mode across those
123
+ 24 responses; this measures bytes, **not agent token usage or testing quality**.
124
+
125
+ <details>
126
+ <summary><b>Live monitoring from a real local fixture run</b></summary>
127
+
128
+ The dashboard below captured a saved-name persistence failure, one independent
129
+ reproduction receipt and two exercised goals. It shows actual runtime events,
130
+ not model reasoning or a simulated percentage. MCP hosts decide whether to display
131
+ progress notifications; the local dashboard works independently of that UI choice.
132
+
133
+ <img src="https://raw.githubusercontent.com/chriswu727/argus/main/assets/agent-monitor.png" alt="Argus live dashboard with two exercised goals, one persistence check and one reproduced symptom" width="800">
134
+
135
+ Reproduce without a model: run `pytest tests/test_monitoring.py` after installing
136
+ development dependencies and Chromium.
137
+
138
+ </details>
139
+
140
+ ## Evidence and reports
77
141
 
78
142
  Give it a URL; get a report of bugs — each tagged with whether Argus **independently reproduced it** or only observed it:
79
143
 
@@ -81,7 +145,10 @@ Give it a URL; get a report of bugs — each tagged with whether Argus **indepen
81
145
  <img src="https://raw.githubusercontent.com/chriswu727/argus/main/assets/report.png" alt="Argus bug report — verified findings with reproduction receipts" width="800">
82
146
  </div>
83
147
 
84
- The green badge is the whole point. Anyone can have an LLM *claim* a bug. Argus re-loads the page from scratch and re-checks the symptom before it says **VERIFIED** — so the report is a list of bugs you can trust, not a list of guesses to triage.
148
+ A reproduction receipt shows whether a specified symptom survived an independent
149
+ recheck. It does not, by itself, prove a root cause or that the behavior violates
150
+ the product requirements. Observations, inconclusive checks and reproduced
151
+ symptoms remain distinguishable in the agent result and exported evidence.
85
152
 
86
153
  ---
87
154
 
@@ -92,13 +159,13 @@ flowchart LR
92
159
  A(["observe"]) --> B{"looks wrong?"}
93
160
  B -->|not sure| C["act: click · type · resize · verify"]
94
161
  C --> A
95
- B -->|bug| D["verify_persistence — reload from a clean state"]
96
- D -->|symptom repeats| E(["VERIFIED"])
97
- D -->|symptom gone or uncertain| F(["not certified"])
98
- E --> G[["report: HTML · JSON · JUnit · SARIF"]]
162
+ B -->|suspected defect| D["verify_persistence — inspect a fresh load"]
163
+ D --> E["record_bug with an independent recheck"]
164
+ E --> F(["reproduced / not reproduced / inconclusive"])
165
+ F --> G[["report: HTML · JSON · JUnit · SARIF"]]
99
166
  ```
100
167
 
101
- The agent is the intelligence. Argus supplies concise QA guidance, a description-keyed tool surface (`click_what("Login button")`, not `click(7)`), a goal coverage ledger, and a **reproduction-receipt engine** that turns "the model thinks this is a bug" into "this bug is real, here's the proof."
168
+ The agent supplies the judgment. Argus supplies concise QA guidance, a description-keyed tool surface (`click_what("Login button")`, not `click(7)`), a goal coverage ledger, and a **reproduction-receipt engine** that distinguishes observed, reproduced and inconclusive symptoms.
102
169
 
103
170
  A receipt confirms the specified observable symptom, not every possible root
104
171
  cause. A clean load preserves authentication and browser storage by default;
@@ -156,6 +223,31 @@ The button adds Argus to Cursor; run the Chromium installation command above onc
156
223
 
157
224
  The default `core` profile exposes the primary web-testing workflow without flooding the host with every specialist tool. Use `uvx --from argus-testing argus-mcp --list-tools` to inspect the selected profile, `--tool-profile screen` for native macOS testing, or `--tool-profile full` for the entire advanced surface. `ARGUS_TOOL_PROFILE` provides the same setting through the environment.
158
225
 
226
+ For agents that read structured MCP results, use the focused configuration:
227
+
228
+ ```bash
229
+ uvx --from argus-testing argus-mcp --tool-profile agent --response-mode compact --monitor-port 8765
230
+ ```
231
+
232
+ Read returned observations before issuing another `observe`. Batch related goals
233
+ with `coverage_update(goal=["edit profile", "reload profile"], status="in_progress")`;
234
+ finish them with observed evidence, not an assumed pass. The monitor stays local.
235
+
236
+ `--response-mode delta` is an additional opt-in for clients that can retain and
237
+ merge snapshot IDs. Keep `compact` if the host cannot reliably resolve bases.
238
+ See the [snapshot contract](docs/agent-protocol.md#incremental-observations).
239
+
240
+ To check installation, product version, stdio handshake, discovery and structured
241
+ errors without opening a browser or calling a model:
242
+
243
+ ```bash
244
+ uvx --from argus-testing python -m argus.install_check
245
+ ```
246
+
247
+ Editable installs must be reinstalled after a version change so their package
248
+ metadata matches the source. MCP `serverInfo.version` now reports Argus's version,
249
+ not the MCP SDK version.
250
+
159
251
  Then just ask, in your agent session:
160
252
 
161
253
  > **"Test my app at http://localhost:3000 — find real bugs."**
@@ -247,9 +339,9 @@ selected URL is printed to **stderr**, so stdio MCP messages stay intact.
247
339
  | `/` | Local live dashboard |
248
340
  | `/healthz` | HTTP process liveness |
249
341
  | `/readyz` | Runtime initialized and not shutting down; 503 otherwise |
250
- | `/api/status` | Versioned state snapshot, counts and capture drops |
342
+ | `/api/status` | Versioned state snapshot, counts, capture drops and numeric overhead |
251
343
  | `/api/events` | Server-Sent Events, `Last-Event-ID` reconnect and gap notices |
252
- | `/metrics` | Prometheus text counters and operation duration sums/counts |
344
+ | `/metrics` | Prometheus operation, response-byte and component-timing counters |
253
345
 
254
346
  The monitor binds only to `127.0.0.1`, rejects unrelated Host/Origin headers,
255
347
  and has no action endpoints. It emits no page contents, URLs, typed values,
@@ -295,32 +387,35 @@ application data; see [Privacy](PRIVACY.md).
295
387
 
296
388
  ## Why Argus is different
297
389
 
298
- **Existing testing tools only test what you script.** Playwright and Cypress run the assertions you wrote. Argus *discovers* bugs you didn't think to test for — and then does the thing an LLM alone can't be trusted to do: **proves them.**
390
+ **Give exploratory testing a repeatable evidence workflow.** An agent can use
391
+ Argus to investigate journeys beyond prewritten checks, inspect what changed,
392
+ and independently recheck suspected symptoms. The agent still decides whether
393
+ the observed behavior violates the application's requirements; a reproduction
394
+ receipt is supporting evidence, not a guarantee of correctness or zero false positives.
299
395
 
300
396
  | | |
301
397
  |---|---|
302
- | **Autonomous & black-box** | You give it a URL, not a test plan. It explores like a real user — no repo access, no scripted steps. |
398
+ | **Agent-directed & black-box** | Give your agent a URL and scope. It explores through Argus without requiring application source access or prewritten test scripts. |
303
399
  | **Coverage contract** | Optional natural-language goals, user constraints, discovered pages, and time budget stay visible throughout the session and in the final report. |
304
- | **Reproduction receipts** | Before certifying a bug, it re-loads the page from a clean state and re-confirms the symptom. Engineered for **zero false-certifications.** |
305
- | **Finds human-eye bugs** | Fake "Only 3 left!" scarcity, a "Saved" toast that doesn't save, a sale badge where the price didn't drop, a stale navbar after a rename. Static analysis catches none of these. |
306
- | **Discover → guard** | Findings are journaled; `argus-regression` re-checks them on every build with **zero LLM cost** and a non-zero exit — a real CI gate against known bugs coming back. |
307
- | **Machine-readable** | Every report also emits JSON, JUnit, and SARIF — so findings gate a pipeline and surface as inline **GitHub PR annotations.** |
400
+ | **Reproduction receipts** | Independent checks distinguish reproduced, not-reproduced and inconclusive symptoms. Reloads preserve authentication and storage by default; replay can repeat writes. |
401
+ | **Runtime evidence** | Inspect misleading success messages, cross-page inconsistencies and persistence failures in the running app. The agent interprets this evidence against the intended behavior. |
402
+ | **Discover → guard** | Add `argus-regression` to CI to recheck eligible journaled findings without model calls. Exit codes distinguish reproduced symptoms, operational failures and inconclusive checks. |
403
+ | **Machine-readable** | JSON, JUnit and SARIF exports support downstream processing. CI gates and code-scanning uploads require integration; URL-based findings do not guarantee source-line PR annotations. |
308
404
 
309
405
  ---
310
406
 
311
407
  ## How it compares
312
408
 
313
- On the axis that matters for finding bugs — *autonomously discover, independently verify, and report* — Argus occupies a different slot from the browser-MCP crowd:
314
-
315
- | | **Argus** | Playwright MCP | Chrome DevTools MCP | browser-use |
316
- |---|:---:|:---:|:---:|:---:|
317
- | Autonomously finds unknown bugs | Yes | No *(driver)* | No *(debugger)* | Partial *(task-scoped)* |
318
- | Independently verifies each finding | Yes *(receipt)* | No | No | No *(LLM score)* |
319
- | Evidence-rich bug report | Yes | No | No | Partial |
320
- | Black-box (no repo / source access) | Yes | Yes | Yes | Yes |
321
- | Zero-LLM CI regression gate | Yes | Partial | No | Partial |
409
+ Use Argus when your agent can operate a browser but needs a repeatable QA
410
+ workflow: track unfinished journeys, check saved/deleted state after reload,
411
+ reproduce a reported symptom, and preserve evidence for a regression check.
412
+ It complements general browser automation and debugging tools; it does not
413
+ replace the host model's judgment or guarantee discovery of every defect.
322
414
 
323
- > These aren't "worse" tools — they're a different job. Playwright MCP gives an agent excellent hands; Chrome DevTools MCP gives it deep network/perf/memory inspection Argus doesn't have. Argus is the layer that *decides what's a bug and proves it.* Use them together.
415
+ The [paired evaluation](docs/evaluation.md) compares a conventional Playwright
416
+ MCP adapter with Argus using the same host and tasks. That adapter is not the
417
+ official Playwright MCP package, and these results must not be presented as a
418
+ ranking of third-party products.
324
419
 
325
420
  ---
326
421
 
@@ -338,14 +433,19 @@ $ python -m argus.bench --target all
338
433
  `34 / 34` is the **capability ceiling** — what's *findable* through the tool surface, measured by deterministic scripts. It is deliberately separate from *how often a given LLM remembers to use the tools well*, which is noisy and honestly reported below.
339
434
 
340
435
  <details>
341
- <summary><b>Real-LLM recall — the honest number (and why we report the spread)</b></summary>
436
+ <summary><b>Real-agent evaluation</b></summary>
342
437
 
343
- `python -m argus.bench.agent_runner` puts an **actual model** in the driver's seat and scores recall across trials. What we've learned running it:
438
+ The opt-in Codex runner uses real stdio MCP calls against eight synthetic
439
+ cases: profile persistence, deletion, validation/recovery and
440
+ cross-page totals. It keeps model/host settings fixed, isolates each run and
441
+ caps calls and elapsed time. A separate trace-and-server-oracle review adjudicates
442
+ findings and coverage before the scorer computes paired metrics.
443
+ Read the [published 0.8 results and limitations](docs/evaluation-0.8.md), including
444
+ the residual defect in the nominally fixed form fixture and all invalid preflights.
344
445
 
345
- 1. **Real recall sits well below the `34/34` ceiling.** A live driver finds a fraction of the seeded bugs per pass — the ceiling is what's *findable*, this is what a model *finds*.
346
- 2. **Variance is large — never rank models on a few runs.** Per-trial recall swings widely; we report the spread, not a single hero number.
347
- 3. **Dogfooding the bench found real bugs in Argus itself** — a `record_bug` crash on a string argument that silently dropped findings, resolver misses on common phrasings. The tool-testing tool got tested.
348
- 4. **Precision holds regardless of driver.** Across every trial, the reproduction receipt kept false-certifications at zero — a weak model finds fewer bugs, but the ones marked VERIFIED are still real.
446
+ The older `argus.bench.agent_runner` uses heuristic keyword matching. It is a
447
+ legacy exploratory diagnostic, not independent precision/recall evidence and
448
+ not the basis for current product claims. See [methodology](docs/evaluation.md).
349
449
 
350
450
  </details>
351
451
 
@@ -354,7 +454,7 @@ $ python -m argus.bench --target all
354
454
 
355
455
  **BuggyTasks** (`:5555`) — 22 mechanical bugs in a task app: console errors, dead links, fake delete (UI says "deleted!" but data persists on refresh), auth bypass, NaN dates, off-by-one counts, race conditions. The "scripted E2E could find these" tier.
356
456
 
357
- **DarkShop** (`:5556`) — 12 human-eye bugs in a polished-looking store: hardcoded "Only 3 left!" scarcity, `-50%` badges where sale price equals original, a "free shipping over $50" banner contradicted by a flat `$5` at checkout, inverted visual hierarchy ("Add to Cart" demoted under a prominent "Subscribe"), cross-page state drift (rename sticks on /account, navbar greeting doesn't). **Static analysis catches roughly none of these** — they require an agent that reads the page and *reasons.*
457
+ **DarkShop** (`:5556`) — 12 seeded runtime and visual scenarios in a polished-looking store: hardcoded "Only 3 left!" scarcity, `-50%` badges where sale price equals original, a "free shipping over $50" banner contradicted by a flat `$5` at checkout, inverted visual hierarchy ("Add to Cart" demoted under a prominent "Subscribe"), cross-page state drift (rename sticks on /account, navbar greeting doesn't). These fixtures exercise comparisons between visible claims and observed behavior; they are not a benchmark against static analysis tools.
358
458
 
359
459
  ```bash
360
460
  python test-site/app.py # BuggyTasks :5555
@@ -379,6 +479,7 @@ uvx --from argus-testing argus-mcp --tool-profile full --list-tools
379
479
  | Profile | Public tools | Intended use |
380
480
  |---------|-------------:|--------------|
381
481
  | `core` | 30 | Primary browser QA workflow; the default. |
482
+ | `agent` | 19 | Focused browser journeys, concise descriptions and batch coverage; pair with compact responses when the host supports structured results. |
382
483
  | `screen` | 14 | Focused native macOS testing through Accessibility and screenshots. |
383
484
  | `full` | 77 | Everything in core and screen, plus specialist browser, state, network, coordinate, and crawl controls. |
384
485
 
@@ -400,11 +501,11 @@ uvx --from argus-testing argus-mcp --tool-profile full --list-tools
400
501
  | `screenshot` · `screenshot_diff` | Capture viewport, full-page, or element evidence and produce a red-tint pixel-diff overlay. |
401
502
  | `get_errors` | Drain correlated console errors and HTTP 4xx/5xx events captured since the previous read. |
402
503
  | `capsule_save` · `capsule_restore` | Save and restore a named authenticated or seeded browser state, with an optional liveness check. |
403
- | `verify_persistence` | Force a fresh load and check whether target text is present or absent. The “Saved!” toast is not proof; this is. |
504
+ | `verify_persistence` | Force a fresh load and return target presence plus the page observation. This checks persistence, not semantic correctness. |
404
505
  | `test_action` · `test_form` | Perform a description-keyed action or form submission and return the resulting state diff in one round trip. |
405
506
  | `check_links` · `check_performance` | Probe current-page internal links and expose raw browser performance metrics without auto-certifying generic audit findings. |
406
507
  | `regression_check` | Re-test journaled findings for the current origin without requiring another discovery pass. |
407
- | `record_bug` · `record_observation` | Record a reproducible defect with evidence and receipt, or keep a qualitative review note separate from certified bugs. |
508
+ | `record_bug` · `record_observation` | Record a finding with evidence and its recheck outcome, or keep a qualitative review note separate. Inspect the receipt rather than treating recording as confirmation. |
408
509
  | `end_session` | Close the active session and emit HTML, JSON, JUnit, and SARIF reports. |
409
510
 
410
511
  </details>
@@ -491,7 +592,11 @@ necessary and the host is authorized to receive credentials.
491
592
  <details>
492
593
  <summary><b>Trust the agent, don't simulate intelligence</b></summary>
493
594
 
494
- Argus assumes an Opus-class driver. Static rules that pretend to *be* the smart layer are subtractive — they add maintenance and false positives and pull attention from what the agent actually saw. So `detector.py` is tiny: it only captures the two channels the agent literally cannot see (the console event stream and the HTTP layer). "Is this toast misleading? Is the visual hierarchy wrong? Is that count off?" — the agent reads `observe()` and decides.
595
+ The host agent makes semantic judgments. Argus supplies bounded observations,
596
+ deterministic checks, evidence-linked coverage and explicit uncertainty.
597
+ Guidance suggests applicable probes; it never silently executes them or treats
598
+ a heuristic as a confirmed defect. Browser events are captured as observations,
599
+ not proof that the user-facing behavior is wrong.
495
600
 
496
601
  </details>
497
602
 
@@ -528,7 +633,8 @@ argus/
528
633
  ├── reporter.py # atomic report orchestration and exporter registry
529
634
  ├── resolver.py # description → element (web + screen)
530
635
  ├── cli.py # autonomous exploration and shared regression entry point
531
- └── bench/ # deterministic capability + optional real-LLM harness
636
+ ├── evaluation/ # paired schemas, isolated fixtures, opt-in host runner, adjudication
637
+ └── bench/ # deterministic capability + legacy heuristic harness
532
638
  test-site/ # BuggyTasks (22 mechanical bugs)
533
639
  human-eye-fixture/ # DarkShop (12 human-eye bugs)
534
640
  ```
@@ -51,6 +51,11 @@ inputs require approved directories in `ARGUS_FILE_ROOTS`; see [Security](SECURI
51
51
  model service.
52
52
  - Installing or upgrading through PyPI, `pip`, or `uvx` contacts those package
53
53
  distribution services under their own policies.
54
+ - The optional `argus.evaluation.codex_runner` sends synthetic testing prompts
55
+ and tool evidence to the operator's ChatGPT-backed Codex session only after
56
+ explicit subscription-usage opt-in. It stores local answers, usage, tool traces
57
+ and fixture oracle logs, not model reasoning. Raw traces can contain local
58
+ paths and should remain private. Normal MCP operation never starts this runner.
54
59
 
55
60
  Argus does not intentionally transmit test artifacts to the project maintainer.
56
61
  Review reports and screenshots before sharing them because they can contain