argus-testing 0.7.1__tar.gz → 0.9.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {argus_testing-0.7.1 → argus_testing-0.9.0}/.gitignore +3 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/AGENTS.md +30 -9
- {argus_testing-0.7.1 → argus_testing-0.9.0}/CHANGELOG.md +41 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/CONTRIBUTING.md +18 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/PKG-INFO +146 -40
- {argus_testing-0.7.1 → argus_testing-0.9.0}/PRIVACY.md +5 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/README.md +144 -38
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/assets/dashboard.html +1 -1
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/assets/dashboard.js +4 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/monitoring.py +8 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/actions.py +32 -13
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/coverage.py +3 -3
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/expectations.py +70 -11
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/findings.py +55 -11
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/finish.py +19 -2
- argus_testing-0.9.0/argus/application/guidance.py +105 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/journeys.py +45 -9
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/lifecycle.py +14 -5
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/manager.py +64 -28
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/navigation.py +11 -3
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/observation.py +64 -41
- argus_testing-0.9.0/argus/application/payloads.py +49 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/probes.py +19 -3
- argus_testing-0.9.0/argus/application/progress.py +72 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/session.py +3 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/browser.py +10 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/domain/contracts.py +11 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/entrypoints/mcp/catalog.py +22 -0
- argus_testing-0.9.0/argus/entrypoints/mcp/overhead.py +40 -0
- argus_testing-0.9.0/argus/entrypoints/mcp/results.py +98 -0
- argus_testing-0.9.0/argus/entrypoints/mcp/server.py +134 -0
- argus_testing-0.9.0/argus/entrypoints/mcp/snapshots.py +77 -0
- argus_testing-0.9.0/argus/evaluation/__init__.py +1 -0
- argus_testing-0.9.0/argus/evaluation/__main__.py +68 -0
- argus_testing-0.9.0/argus/evaluation/adjudication.py +114 -0
- argus_testing-0.9.0/argus/evaluation/agent_fixture.py +156 -0
- argus_testing-0.9.0/argus/evaluation/browser_baseline.py +89 -0
- argus_testing-0.9.0/argus/evaluation/codex_runner.py +344 -0
- argus_testing-0.9.0/argus/evaluation/fixture.py +59 -0
- argus_testing-0.9.0/argus/evaluation/gateway.py +111 -0
- argus_testing-0.9.0/argus/evaluation/models.py +117 -0
- argus_testing-0.9.0/argus/evaluation/overhead.py +121 -0
- argus_testing-0.9.0/argus/evaluation/scoring.py +94 -0
- argus_testing-0.9.0/argus/evaluation/smoke.py +153 -0
- argus_testing-0.9.0/argus/install_check.py +68 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/mcp_server.py +7 -2
- argus_testing-0.9.0/argus/measurement.py +61 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/reporter.py +2 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/settings.py +5 -0
- argus_testing-0.9.0/docs/agent-protocol.md +176 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/docs/architecture.md +45 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/docs/decisions.md +66 -0
- argus_testing-0.9.0/docs/evaluation-0.8.md +122 -0
- argus_testing-0.9.0/docs/evaluation.md +208 -0
- argus_testing-0.9.0/examples/agent-evaluation-optimized-evidence.json +4008 -0
- argus_testing-0.9.0/examples/agent-evaluation-optimized.json +939 -0
- argus_testing-0.9.0/examples/agent-evaluation-pilot-evidence.json +2349 -0
- argus_testing-0.9.0/examples/agent-evaluation-pilot.json +409 -0
- argus_testing-0.9.0/examples/response-overhead.json +170 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/llms-install.md +22 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/pyproject.toml +3 -2
- {argus_testing-0.7.1 → argus_testing-0.9.0}/scripts/check_distributions.py +3 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/server.json +4 -4
- argus_testing-0.9.0/tests/test_adjudication.py +104 -0
- argus_testing-0.9.0/tests/test_agent_results.py +155 -0
- argus_testing-0.9.0/tests/test_agent_workflow.py +172 -0
- argus_testing-0.9.0/tests/test_codex_evaluation.py +154 -0
- argus_testing-0.9.0/tests/test_evaluation.py +193 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_expectation_oracle.py +4 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_mcp_contract.py +3 -1
- argus_testing-0.9.0/tests/test_monitoring.py +198 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_release_metadata.py +16 -2
- argus_testing-0.9.0/tests/test_response_efficiency.py +235 -0
- argus_testing-0.9.0/tests/test_stdio_protocol.py +132 -0
- argus_testing-0.7.1/argus/entrypoints/mcp/server.py +0 -74
- argus_testing-0.7.1/tests/test_monitoring.py +0 -111
- argus_testing-0.7.1/tests/test_stdio_protocol.py +0 -54
- {argus_testing-0.7.1 → argus_testing-0.9.0}/LICENSE +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/SECURITY.md +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/__init__.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/_compat.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/access.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/__init__.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/assets/__init__.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/assets/dashboard.css +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/journal.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/logging.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/reporting/__init__.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/reporting/common.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/reporting/html.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/reporting/json.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/reporting/junit.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/reporting/privacy.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/reporting/sarif.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/scripts/__init__.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/scripts/check_layout_js.js +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/scripts/extract_elements_js.js +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/scripts/extract_page_content_js.js +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/scripts/extract_performance_js.js +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/scripts/find_visible_element_js.js +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/adapters/scripts/inspect_element_js.js +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/__init__.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/capsules.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/capture.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/context.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/events.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/evidence.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/inspection.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/network_storage.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/operations.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/regression.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/screen.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/screenshots.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/telemetry.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/text.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/verification.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/application/verifiers.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/bench/__init__.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/bench/__main__.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/bench/agent_runner.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/bench/runner.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/bench/scenarios_buggytasks.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/bench/scenarios_darkshop.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/bootstrap.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/cleanup.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/cli.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/config.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/detector.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/differ.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/domain/__init__.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/domain/detector.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/domain/models.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/domain/replay.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/domain/verification.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/entrypoints/__init__.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/entrypoints/mcp/__init__.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/explorer.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/models.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/planner.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/ports.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/resolver.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/runtime.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/screen/__init__.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/screen/backend.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/screen/permissions.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/screen/safety.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/screen/validate.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/security.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/argus/storage.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/docs/implementation-0.7.md +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/examples/focus.yaml +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/examples/mcp-config.json +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/glama.json +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/human-eye-fixture/app.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/test-site/app.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/__init__.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/conftest.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_action_targeting.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_architecture_runtime.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_bench_report.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_browser_shadow.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_cross_stack.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_detector_console.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_detector_network.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_distribution_contents.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_final_hardening.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_journal_regression.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_regression_cli.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_replay_minimize.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_replay_receipt.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_replay_trace.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_reporter_badge.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_reproduction_receipt.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_resolver.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_runtime_limits.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_screen_permissions.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_screen_safety.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_security_boundaries.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_session_lifecycle.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_state_capsules.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_storage_cleanup.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_substrate.py +0 -0
- {argus_testing-0.7.1 → argus_testing-0.9.0}/tests/test_visual_inspection.py +0 -0
|
@@ -25,7 +25,18 @@ argus-mcp --list-tools # show the selected profile and public tools
|
|
|
25
25
|
argus-mcp --doctor # macOS Screen Recording + Accessibility check
|
|
26
26
|
```
|
|
27
27
|
|
|
28
|
-
The default profile is `core`.
|
|
28
|
+
The default profile is `core`. For structured-result-aware hosts, use
|
|
29
|
+
`--tool-profile agent --response-mode compact` for 19 focused tools.
|
|
30
|
+
|
|
31
|
+
`--response-mode delta` is opt-in for snapshot-aware clients. Only observation
|
|
32
|
+
fields listed in the protocol may be inherited from an exact `base_id`; action
|
|
33
|
+
and verification results are current-call-only. If context or a base is missing,
|
|
34
|
+
call `observe` for full state. Never infer a passing check from unchanged fields.
|
|
35
|
+
Use `python -m argus.install_check` to verify installed versions and stdio without
|
|
36
|
+
a browser or model, and `python -m argus.evaluation.overhead` for deterministic
|
|
37
|
+
byte measurements. Bytes are not model tokens. Monitoring retains numeric
|
|
38
|
+
overhead only; do not add page content or arguments to metrics.
|
|
39
|
+
Use `--tool-profile screen` for native
|
|
29
40
|
macOS review and `--tool-profile full` only when the broader network,
|
|
30
41
|
storage, tabs, crawl, and coordinate surface is needed. The
|
|
31
42
|
`ARGUS_TOOL_PROFILE` environment variable provides the same setting.
|
|
@@ -48,6 +59,16 @@ the user's broader task. The short version:
|
|
|
48
59
|
Map → **Use it** (walk each goal end-to-end, carrying real state) →
|
|
49
60
|
Hypothesize → Act → Observe → Verify → Record → Cover.
|
|
50
61
|
|
|
62
|
+
## Structured results (0.8)
|
|
63
|
+
|
|
64
|
+
Read `structuredContent.data`
|
|
65
|
+
for operation-specific evidence where available; text and images remain for
|
|
66
|
+
compatibility. `status: completed` means the tool ran, not that the application
|
|
67
|
+
passed. Inspect expectation matches, finding verdicts and export warnings.
|
|
68
|
+
Progress notifications describe actual phases, not percentages or proof of
|
|
69
|
+
coverage. Missing live notifications do not mean execution stopped; the final
|
|
70
|
+
result includes captured stages. See [the protocol](docs/agent-protocol.md).
|
|
71
|
+
|
|
51
72
|
## Tools you'll use most
|
|
52
73
|
|
|
53
74
|
### Web mode
|
|
@@ -55,13 +76,13 @@ the user's broader task. The short version:
|
|
|
55
76
|
| Tool | Purpose |
|
|
56
77
|
|------|---------|
|
|
57
78
|
| `start_session(url, review_mode=..., goals=[...], constraints=[...], time_budget_minutes=...)` | Launch Playwright, establish the review contract, and return the one-time QA protocol plus initial observation. The budget is advisory; `0` means none. |
|
|
58
|
-
| `observe()` | URL
|
|
59
|
-
| `coverage_update(goal, status, evidence)` |
|
|
79
|
+
| `observe()` | URL, targets, feedback, counts, coverage and next-check hints. Reuse post-action observations; call again only for missing or stale evidence. |
|
|
80
|
+
| `coverage_update(goal, status, evidence)` | Accepts one goal or a list of related goals. Mark `in_progress` before the shared journey. `exercised` and `blocked` require observed evidence and link URLs, value-redacted actions, screenshots, checks and findings. Invalid batches make no changes. |
|
|
60
81
|
| `click_what(description)` | Click the element matching `description`. Returns the top candidates if ambiguous — rephrase rather than guess. |
|
|
61
82
|
| `type_into(description, text)` / `select_into(description, value)` | Inputs and dropdowns by description. |
|
|
62
83
|
| `test_action(target, expect=...)` | Click + before/after diff in one call. Pass `expect` to PREDICT the outcome ({"count":{"label":"tasks","delta":1}}, {"gains":"Buy milk"}, {"removes":...}, {"text_present":...}, {"toast":...}, {"url_changed":true}) and Argus reports MATCH / SURPRISE — a surprise is a bug lead. Also shows CROSS-STACK: which requests the click fired (methods/statuses) and a CHECK nudge when a message appeared without a matching write. |
|
|
63
|
-
| `verify_persistence(expect, target_text, after_url)` | Forces a fresh GET;
|
|
64
|
-
| `capsule_save(name, liveness_marker)` / `capsule_restore(name)` | Snapshot
|
|
84
|
+
| `verify_persistence(expect, target_text, after_url)` | Forces a fresh GET; returns target presence and the page observation. Check persistence once, repeat only to resolve uncertainty. A matching symptom is not proof of a semantic defect. |
|
|
85
|
+
| `capsule_save(name, liveness_marker)` / `capsule_restore(name)` | Snapshot cookies and page storage after establishing state through the UI. Restore replaces the captured client state and checks a saved liveness marker when provided; without one, validity remains unverified. It does not reset backend data or guarantee identical application state across journeys. |
|
|
65
86
|
| `regression_check()` | Re-test the findings journaled in prior runs against the CURRENT build: STILL-PRESENT / NO-LONGER-REPRODUCES / INCONCLUSIVE. "Did my fix land, did anything come back?" |
|
|
66
87
|
| `inspect_element(description)` | Computed styles + ARIA + outerHTML + truncation flag for interactive or visible non-interactive content. |
|
|
67
88
|
| `check_layout()` | Bounded viewport signals for horizontal overflow, clipped text, small targets, and fixed/sticky overlays. |
|
|
@@ -91,16 +112,16 @@ the user's broader task. The short version:
|
|
|
91
112
|
```
|
|
92
113
|
start_session(url, goals=[...], constraints=[...], time_budget_minutes=15)
|
|
93
114
|
# initial result already MAPS the page
|
|
94
|
-
coverage_update(
|
|
115
|
+
coverage_update([goal_a, goal_b], "in_progress")
|
|
95
116
|
# USE IT — pick a real goal, walk it
|
|
96
117
|
# end-to-end, carrying state across pages
|
|
97
118
|
# HYPOTHESIZE — what could go wrong
|
|
98
119
|
click_what(...) / type_into(...) / # ACT — one probe per call
|
|
99
120
|
test_form({...})
|
|
100
|
-
|
|
121
|
+
# read returned observation; observe only if needed
|
|
101
122
|
verify_persistence(...) # VERIFY (delete / save / submit / toggle)
|
|
102
123
|
record_bug(..., verify={...}) # RECORD — verify clause attaches a receipt
|
|
103
|
-
coverage_update(
|
|
124
|
+
coverage_update([goal_a, goal_b], "exercised",
|
|
104
125
|
evidence)
|
|
105
126
|
... repeat ...
|
|
106
127
|
end_session() # reports completed + unfinished coverage
|
|
@@ -157,7 +178,7 @@ Don't record speculation, polish nits, or static a11y / SEO that
|
|
|
157
178
|
- `argus-mcp --version` — what version your MCP host actually loaded.
|
|
158
179
|
- `argus-mcp --doctor` — macOS permission probes for screen mode.
|
|
159
180
|
- `argus-mcp --list-tools` — selected tool profile and exported names.
|
|
160
|
-
- `argus-mcp --tool-profile core|screen|full` — control discovery size.
|
|
181
|
+
- `argus-mcp --tool-profile agent|core|screen|full` — control discovery size.
|
|
161
182
|
- `argus-mcp --unsafe` — turn on `eval_js` (off by default).
|
|
162
183
|
- `argus-mcp --monitor-port 8765` — enable the read-only loopback dashboard,
|
|
163
184
|
health/readiness endpoints, SSE events and metrics; use `0` for a free port.
|
|
@@ -1,5 +1,46 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.9.0 - 2026-10-25
|
|
4
|
+
|
|
5
|
+
- Report the Argus product version in the MCP handshake instead of the SDK
|
|
6
|
+
version. Add a no-browser installed-package stdio check and run it against
|
|
7
|
+
fresh release wheels outside the source checkout.
|
|
8
|
+
- Remove redundant output-schema display titles without changing validation
|
|
9
|
+
constraints or tool display titles.
|
|
10
|
+
- Add opt-in delta observations with explicit base IDs, field removals, full
|
|
11
|
+
resynchronization, lifecycle invalidation and bounded redacted retention.
|
|
12
|
+
Preserve errors, current-call verdicts, image blocks and compatibility defaults.
|
|
13
|
+
- Measure response bytes and instrumented browser waits, observation, screenshots,
|
|
14
|
+
report export and encoding. Expose numeric totals through loopback status and
|
|
15
|
+
Prometheus; keep page content and input values out of monitoring.
|
|
16
|
+
- Add reproducible, lossless encoding comparisons with navigation controls and
|
|
17
|
+
gateway discovery/response measurements. Do not equate byte savings with model
|
|
18
|
+
token savings or claim a new model effectiveness evaluation.
|
|
19
|
+
|
|
20
|
+
## 0.8.0 - 2026-10-25
|
|
21
|
+
|
|
22
|
+
- Add an optional 19-tool agent profile with concise discovery descriptions,
|
|
23
|
+
compact structured responses, post-action/check observations and atomic batch
|
|
24
|
+
coverage updates. Keep core as the backward-compatible default.
|
|
25
|
+
- Add bounded next-check guidance, persistence reminders and stalled-exploration
|
|
26
|
+
signals. Do not automatically mark goals complete or findings confirmed.
|
|
27
|
+
- Treat missing counts and malformed expectations as unknown, not guessed
|
|
28
|
+
matches. Preserve partial form outcomes and stable finding references.
|
|
29
|
+
- Show real testing phases, goal/check counts and separate finding outcomes in
|
|
30
|
+
the read-only loopback dashboard, without exposing goal text or inputs.
|
|
31
|
+
- Add opt-in, budget-bounded Codex subscription collection, synthetic browser
|
|
32
|
+
fixtures, conventional-browser controls and independent evidence adjudication.
|
|
33
|
+
Model calls are never part of import, normal MCP usage or CI.
|
|
34
|
+
|
|
35
|
+
- Add versioned structured MCP results alongside compatible text/image content,
|
|
36
|
+
including structured evidence for core observation, coverage, verification, findings
|
|
37
|
+
and report flows, with explicit runtime error codes and recovery guidance.
|
|
38
|
+
- Emit bounded, request-local phase progress over MCP and local monitoring;
|
|
39
|
+
retain a final summary when a host does not display notifications.
|
|
40
|
+
- Add paired evaluation schemas and offline metrics with matched controls,
|
|
41
|
+
retained failures, duplicate/false-positive accounting and unknown usage.
|
|
42
|
+
A no-model browser/stdio smoke suite exercises buggy, fixed and clean controls.
|
|
43
|
+
|
|
3
44
|
## 0.7.1 - 2026-10-25
|
|
4
45
|
|
|
5
46
|
- Require Python 3.11 or newer and raise Pillow/filelock/certifi minimum versions to
|
|
@@ -41,6 +41,14 @@ separate macOS manual check; Linux CI does not claim to test that integration.
|
|
|
41
41
|
access `_page`, `_context`, `_locator` or other adapter private state.
|
|
42
42
|
- Monitor additions: `adapters/monitoring.py` and its packaged `assets/`.
|
|
43
43
|
Never add raw inputs, arbitrary URLs, page contents or credentials to events.
|
|
44
|
+
- MCP result schemas: `entrypoints/mcp/results.py`; application evidence uses
|
|
45
|
+
`ToolText` without importing MCP. Keep text/image compatibility and test real stdio.
|
|
46
|
+
- Progress: fixed phases in `application/progress.py`, emitted at actual work
|
|
47
|
+
boundaries through the runtime. Never accept page-supplied progress messages.
|
|
48
|
+
- Coverage hints: `application/guidance.py`; use observed facts and bounded output,
|
|
49
|
+
never model calls, automatic writes or inferred completion. Monitor only counts.
|
|
50
|
+
- Paired evaluation: `evaluation/`; keep model calls and external adjudication
|
|
51
|
+
outside the product runtime. Preserve both arms, failures and unknown usage.
|
|
44
52
|
- Access rules: `access.py`; keep operator policy outside tool-controlled data.
|
|
45
53
|
File tests must explicitly grant their temporary fixture directory.
|
|
46
54
|
- Redaction: `security.py` and the shared reporting privacy wrapper. Test every
|
|
@@ -64,6 +72,16 @@ For the scripted benchmark, start `test-site/app.py` and
|
|
|
64
72
|
a capability ceiling, not a real-agent success rate. Model-provider benchmarks
|
|
65
73
|
consume subscription/API usage and must be explicitly authorized.
|
|
66
74
|
|
|
75
|
+
Run `python -m argus.evaluation --smoke` for the isolated, no-model paired
|
|
76
|
+
protocol check. See [evaluation methodology](docs/evaluation.md) before reporting
|
|
77
|
+
agent effectiveness. Protocol details are in [agent protocol](docs/agent-protocol.md).
|
|
78
|
+
|
|
79
|
+
The opt-in `evaluation.codex_runner` consumes subscription usage and requires
|
|
80
|
+
explicit authorization. Never add it to CI. Unit and stdio gateway tests do not
|
|
81
|
+
invoke a model. Test every fixture's buggy/fixed ground truth through real browser
|
|
82
|
+
interactions before collecting model trials. Keep raw collections and credentials
|
|
83
|
+
outside the public repository; only reviewed synthetic aggregate evidence belongs here.
|
|
84
|
+
|
|
67
85
|
For release, synchronize `pyproject.toml` and both version fields in
|
|
68
86
|
`server.json`, update `CHANGELOG.md`, build with `python -m build`, check with
|
|
69
87
|
`twine check`, and smoke-test the wheel outside the source directory. JS and
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: argus-testing
|
|
3
|
-
Version: 0.
|
|
4
|
-
Summary:
|
|
3
|
+
Version: 0.9.0
|
|
4
|
+
Summary: Agent-first testing MCP with evidence-backed coverage, persistence checks and reproduction receipts
|
|
5
5
|
Project-URL: Homepage, https://yichenwu.dev/projects/argus
|
|
6
6
|
Project-URL: Documentation, https://github.com/chriswu727/argus#readme
|
|
7
7
|
Project-URL: Repository, https://github.com/chriswu727/argus
|
|
@@ -56,9 +56,9 @@ Description-Content-Type: text/markdown
|
|
|
56
56
|
|
|
57
57
|
# Argus
|
|
58
58
|
|
|
59
|
-
**
|
|
59
|
+
**A testing-engineer toolkit for AI agents—explore user journeys, check outcomes, and reproduce bug symptoms with evidence, through MCP.**
|
|
60
60
|
|
|
61
|
-
Argus is an [MCP](https://modelcontextprotocol.io/) server
|
|
61
|
+
Argus is an [MCP](https://modelcontextprotocol.io/) server that gives your coding agent a testing workflow: evidence-backed coverage, action-and-observation tools, persistence checks, independent reproduction receipts, and live progress. The host agent decides what to investigate; Argus helps it check what actually happened. Browser QA works across platforms, with a separate native macOS tool profile.
|
|
62
62
|
|
|
63
63
|
[](https://pypi.org/project/argus-testing/)
|
|
64
64
|
[](https://pypi.org/project/argus-testing/)
|
|
@@ -67,13 +67,77 @@ Argus is an [MCP](https://modelcontextprotocol.io/) server. It adds evidence-fir
|
|
|
67
67
|
[](#benchmarks)
|
|
68
68
|
[](LICENSE)
|
|
69
69
|
|
|
70
|
+
[](https://mcprush.com/to/argus-testing-autonomous-qa-mcp?ref=badge)
|
|
71
|
+
|
|
70
72
|
[Product page](https://yichenwu.dev/projects/argus) · [Quick start](#quick-start) · [Why Argus](#why-argus-is-different) · [Compared](#how-it-compares) · [Tools](#tool-surface) · [Benchmarks](#benchmarks)
|
|
71
73
|
|
|
72
74
|
</div>
|
|
73
75
|
|
|
74
76
|
---
|
|
75
77
|
|
|
76
|
-
##
|
|
78
|
+
## Agent-first testing
|
|
79
|
+
|
|
80
|
+
Argus adds a testing workflow to the agent you already use: inspect, act,
|
|
81
|
+
check persistence, independently reproduce a symptom, and link the evidence.
|
|
82
|
+
HTML reports are an optional way to review the evidence, not the agent interface.
|
|
83
|
+
|
|
84
|
+
Since 0.8.0, tool responses preserve their text and
|
|
85
|
+
images while adding a versioned `structuredContent` envelope and output schema.
|
|
86
|
+
Core actions, observation, verification, findings, coverage and report operations expose
|
|
87
|
+
machine-readable data; other tools currently expose status/error metadata with
|
|
88
|
+
their existing content. Long workflows emit real phase notifications to MCP
|
|
89
|
+
clients that request progress. Clients that do not display them still receive
|
|
90
|
+
the phase summary in the final result. See the [agent protocol](docs/agent-protocol.md).
|
|
91
|
+
|
|
92
|
+
- **Coverage guidance:** up to four evidence-based next checks, outstanding goals,
|
|
93
|
+
persistence reminders, input-boundary probes and stalled-exploration detection.
|
|
94
|
+
- **Action + observation:** click, fill, select, navigate and keyboard actions
|
|
95
|
+
return the current page evidence, reducing the need for a separate observation.
|
|
96
|
+
- **Conservative checks:** missing or malformed evidence stays unknown, not a
|
|
97
|
+
guessed pass or a fabricated bug. Findings link to stable session references.
|
|
98
|
+
- **Focused agent mode:** `argus-mcp --tool-profile agent --response-mode compact`
|
|
99
|
+
exposes 19 common testing tools with concise discovery descriptions, preserves structured evidence,
|
|
100
|
+
and avoids repeating migrated payloads as prose. Compatible mode remains the default.
|
|
101
|
+
- **Visible progress:** real phases, goal counts and check counts over MCP;
|
|
102
|
+
`--monitor-port 8765` adds the local live dashboard without exposing goal text.
|
|
103
|
+
- **Measured response overhead (0.9):** smaller output schemas, numeric payload
|
|
104
|
+
and component-timing counters, and opt-in `--response-mode delta` for clients
|
|
105
|
+
that retain observation snapshots. Missing bases require a full `observe`;
|
|
106
|
+
verification outcomes and image blocks are never replaced by cached verdicts.
|
|
107
|
+
|
|
108
|
+
Does installing Argus improve an agent? Measure it with the
|
|
109
|
+
[paired evaluation harness](docs/evaluation.md): the same model, task, fixture,
|
|
110
|
+
initial state and budget, with both arms and failures retained. The local scripted smoke
|
|
111
|
+
suite validates the harness; it is **not evidence of agent performance gains**.
|
|
112
|
+
|
|
113
|
+
The [0.8 Codex evaluation](docs/evaluation-0.8.md) includes ten matched pairs:
|
|
114
|
+
Argus found 6/6 defect opportunities versus 5/6 for the browser baseline, including
|
|
115
|
+
one post-hoc fixture defect. Both had zero observed false positives and equal
|
|
116
|
+
declared-goal coverage. Argus used more time and tokens. On four repeated pilot
|
|
117
|
+
cases, the targeted update reduced Argus's median calls from 17 to 9; this small
|
|
118
|
+
synthetic sample does not establish general superiority.
|
|
119
|
+
|
|
120
|
+
The no-model [response encoding benchmark](docs/evaluation.md#response-overhead)
|
|
121
|
+
compares identical synthetic evidence, including navigation invalidation. Delta
|
|
122
|
+
mode reduces aggregate result JSON by 47.09% versus compact mode across those
|
|
123
|
+
24 responses; this measures bytes, **not agent token usage or testing quality**.
|
|
124
|
+
|
|
125
|
+
<details>
|
|
126
|
+
<summary><b>Live monitoring from a real local fixture run</b></summary>
|
|
127
|
+
|
|
128
|
+
The dashboard below captured a saved-name persistence failure, one independent
|
|
129
|
+
reproduction receipt and two exercised goals. It shows actual runtime events,
|
|
130
|
+
not model reasoning or a simulated percentage. MCP hosts decide whether to display
|
|
131
|
+
progress notifications; the local dashboard works independently of that UI choice.
|
|
132
|
+
|
|
133
|
+
<img src="https://raw.githubusercontent.com/chriswu727/argus/main/assets/agent-monitor.png" alt="Argus live dashboard with two exercised goals, one persistence check and one reproduced symptom" width="800">
|
|
134
|
+
|
|
135
|
+
Reproduce without a model: run `pytest tests/test_monitoring.py` after installing
|
|
136
|
+
development dependencies and Chromium.
|
|
137
|
+
|
|
138
|
+
</details>
|
|
139
|
+
|
|
140
|
+
## Evidence and reports
|
|
77
141
|
|
|
78
142
|
Give it a URL; get a report of bugs — each tagged with whether Argus **independently reproduced it** or only observed it:
|
|
79
143
|
|
|
@@ -81,7 +145,10 @@ Give it a URL; get a report of bugs — each tagged with whether Argus **indepen
|
|
|
81
145
|
<img src="https://raw.githubusercontent.com/chriswu727/argus/main/assets/report.png" alt="Argus bug report — verified findings with reproduction receipts" width="800">
|
|
82
146
|
</div>
|
|
83
147
|
|
|
84
|
-
|
|
148
|
+
A reproduction receipt shows whether a specified symptom survived an independent
|
|
149
|
+
recheck. It does not, by itself, prove a root cause or that the behavior violates
|
|
150
|
+
the product requirements. Observations, inconclusive checks and reproduced
|
|
151
|
+
symptoms remain distinguishable in the agent result and exported evidence.
|
|
85
152
|
|
|
86
153
|
---
|
|
87
154
|
|
|
@@ -92,13 +159,13 @@ flowchart LR
|
|
|
92
159
|
A(["observe"]) --> B{"looks wrong?"}
|
|
93
160
|
B -->|not sure| C["act: click · type · resize · verify"]
|
|
94
161
|
C --> A
|
|
95
|
-
B -->|
|
|
96
|
-
D
|
|
97
|
-
|
|
98
|
-
|
|
162
|
+
B -->|suspected defect| D["verify_persistence — inspect a fresh load"]
|
|
163
|
+
D --> E["record_bug with an independent recheck"]
|
|
164
|
+
E --> F(["reproduced / not reproduced / inconclusive"])
|
|
165
|
+
F --> G[["report: HTML · JSON · JUnit · SARIF"]]
|
|
99
166
|
```
|
|
100
167
|
|
|
101
|
-
The agent
|
|
168
|
+
The agent supplies the judgment. Argus supplies concise QA guidance, a description-keyed tool surface (`click_what("Login button")`, not `click(7)`), a goal coverage ledger, and a **reproduction-receipt engine** that distinguishes observed, reproduced and inconclusive symptoms.
|
|
102
169
|
|
|
103
170
|
A receipt confirms the specified observable symptom, not every possible root
|
|
104
171
|
cause. A clean load preserves authentication and browser storage by default;
|
|
@@ -156,6 +223,31 @@ The button adds Argus to Cursor; run the Chromium installation command above onc
|
|
|
156
223
|
|
|
157
224
|
The default `core` profile exposes the primary web-testing workflow without flooding the host with every specialist tool. Use `uvx --from argus-testing argus-mcp --list-tools` to inspect the selected profile, `--tool-profile screen` for native macOS testing, or `--tool-profile full` for the entire advanced surface. `ARGUS_TOOL_PROFILE` provides the same setting through the environment.
|
|
158
225
|
|
|
226
|
+
For agents that read structured MCP results, use the focused configuration:
|
|
227
|
+
|
|
228
|
+
```bash
|
|
229
|
+
uvx --from argus-testing argus-mcp --tool-profile agent --response-mode compact --monitor-port 8765
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
Read returned observations before issuing another `observe`. Batch related goals
|
|
233
|
+
with `coverage_update(goal=["edit profile", "reload profile"], status="in_progress")`;
|
|
234
|
+
finish them with observed evidence, not an assumed pass. The monitor stays local.
|
|
235
|
+
|
|
236
|
+
`--response-mode delta` is an additional opt-in for clients that can retain and
|
|
237
|
+
merge snapshot IDs. Keep `compact` if the host cannot reliably resolve bases.
|
|
238
|
+
See the [snapshot contract](docs/agent-protocol.md#incremental-observations).
|
|
239
|
+
|
|
240
|
+
To check installation, product version, stdio handshake, discovery and structured
|
|
241
|
+
errors without opening a browser or calling a model:
|
|
242
|
+
|
|
243
|
+
```bash
|
|
244
|
+
uvx --from argus-testing python -m argus.install_check
|
|
245
|
+
```
|
|
246
|
+
|
|
247
|
+
Editable installs must be reinstalled after a version change so their package
|
|
248
|
+
metadata matches the source. MCP `serverInfo.version` now reports Argus's version,
|
|
249
|
+
not the MCP SDK version.
|
|
250
|
+
|
|
159
251
|
Then just ask, in your agent session:
|
|
160
252
|
|
|
161
253
|
> **"Test my app at http://localhost:3000 — find real bugs."**
|
|
@@ -247,9 +339,9 @@ selected URL is printed to **stderr**, so stdio MCP messages stay intact.
|
|
|
247
339
|
| `/` | Local live dashboard |
|
|
248
340
|
| `/healthz` | HTTP process liveness |
|
|
249
341
|
| `/readyz` | Runtime initialized and not shutting down; 503 otherwise |
|
|
250
|
-
| `/api/status` | Versioned state snapshot, counts
|
|
342
|
+
| `/api/status` | Versioned state snapshot, counts, capture drops and numeric overhead |
|
|
251
343
|
| `/api/events` | Server-Sent Events, `Last-Event-ID` reconnect and gap notices |
|
|
252
|
-
| `/metrics` | Prometheus
|
|
344
|
+
| `/metrics` | Prometheus operation, response-byte and component-timing counters |
|
|
253
345
|
|
|
254
346
|
The monitor binds only to `127.0.0.1`, rejects unrelated Host/Origin headers,
|
|
255
347
|
and has no action endpoints. It emits no page contents, URLs, typed values,
|
|
@@ -295,32 +387,35 @@ application data; see [Privacy](PRIVACY.md).
|
|
|
295
387
|
|
|
296
388
|
## Why Argus is different
|
|
297
389
|
|
|
298
|
-
**
|
|
390
|
+
**Give exploratory testing a repeatable evidence workflow.** An agent can use
|
|
391
|
+
Argus to investigate journeys beyond prewritten checks, inspect what changed,
|
|
392
|
+
and independently recheck suspected symptoms. The agent still decides whether
|
|
393
|
+
the observed behavior violates the application's requirements; a reproduction
|
|
394
|
+
receipt is supporting evidence, not a guarantee of correctness or zero false positives.
|
|
299
395
|
|
|
300
396
|
| | |
|
|
301
397
|
|---|---|
|
|
302
|
-
| **
|
|
398
|
+
| **Agent-directed & black-box** | Give your agent a URL and scope. It explores through Argus without requiring application source access or prewritten test scripts. |
|
|
303
399
|
| **Coverage contract** | Optional natural-language goals, user constraints, discovered pages, and time budget stay visible throughout the session and in the final report. |
|
|
304
|
-
| **Reproduction receipts** |
|
|
305
|
-
| **
|
|
306
|
-
| **Discover → guard** |
|
|
307
|
-
| **Machine-readable** |
|
|
400
|
+
| **Reproduction receipts** | Independent checks distinguish reproduced, not-reproduced and inconclusive symptoms. Reloads preserve authentication and storage by default; replay can repeat writes. |
|
|
401
|
+
| **Runtime evidence** | Inspect misleading success messages, cross-page inconsistencies and persistence failures in the running app. The agent interprets this evidence against the intended behavior. |
|
|
402
|
+
| **Discover → guard** | Add `argus-regression` to CI to recheck eligible journaled findings without model calls. Exit codes distinguish reproduced symptoms, operational failures and inconclusive checks. |
|
|
403
|
+
| **Machine-readable** | JSON, JUnit and SARIF exports support downstream processing. CI gates and code-scanning uploads require integration; URL-based findings do not guarantee source-line PR annotations. |
|
|
308
404
|
|
|
309
405
|
---
|
|
310
406
|
|
|
311
407
|
## How it compares
|
|
312
408
|
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
| Independently verifies each finding | Yes *(receipt)* | No | No | No *(LLM score)* |
|
|
319
|
-
| Evidence-rich bug report | Yes | No | No | Partial |
|
|
320
|
-
| Black-box (no repo / source access) | Yes | Yes | Yes | Yes |
|
|
321
|
-
| Zero-LLM CI regression gate | Yes | Partial | No | Partial |
|
|
409
|
+
Use Argus when your agent can operate a browser but needs a repeatable QA
|
|
410
|
+
workflow: track unfinished journeys, check saved/deleted state after reload,
|
|
411
|
+
reproduce a reported symptom, and preserve evidence for a regression check.
|
|
412
|
+
It complements general browser automation and debugging tools; it does not
|
|
413
|
+
replace the host model's judgment or guarantee discovery of every defect.
|
|
322
414
|
|
|
323
|
-
|
|
415
|
+
The [paired evaluation](docs/evaluation.md) compares a conventional Playwright
|
|
416
|
+
MCP adapter with Argus using the same host and tasks. That adapter is not the
|
|
417
|
+
official Playwright MCP package, and these results must not be presented as a
|
|
418
|
+
ranking of third-party products.
|
|
324
419
|
|
|
325
420
|
---
|
|
326
421
|
|
|
@@ -338,14 +433,19 @@ $ python -m argus.bench --target all
|
|
|
338
433
|
`34 / 34` is the **capability ceiling** — what's *findable* through the tool surface, measured by deterministic scripts. It is deliberately separate from *how often a given LLM remembers to use the tools well*, which is noisy and honestly reported below.
|
|
339
434
|
|
|
340
435
|
<details>
|
|
341
|
-
<summary><b>Real-
|
|
436
|
+
<summary><b>Real-agent evaluation</b></summary>
|
|
342
437
|
|
|
343
|
-
|
|
438
|
+
The opt-in Codex runner uses real stdio MCP calls against eight synthetic
|
|
439
|
+
cases: profile persistence, deletion, validation/recovery and
|
|
440
|
+
cross-page totals. It keeps model/host settings fixed, isolates each run and
|
|
441
|
+
caps calls and elapsed time. A separate trace-and-server-oracle review adjudicates
|
|
442
|
+
findings and coverage before the scorer computes paired metrics.
|
|
443
|
+
Read the [published 0.8 results and limitations](docs/evaluation-0.8.md), including
|
|
444
|
+
the residual defect in the nominally fixed form fixture and all invalid preflights.
|
|
344
445
|
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
4. **Precision holds regardless of driver.** Across every trial, the reproduction receipt kept false-certifications at zero — a weak model finds fewer bugs, but the ones marked VERIFIED are still real.
|
|
446
|
+
The older `argus.bench.agent_runner` uses heuristic keyword matching. It is a
|
|
447
|
+
legacy exploratory diagnostic, not independent precision/recall evidence and
|
|
448
|
+
not the basis for current product claims. See [methodology](docs/evaluation.md).
|
|
349
449
|
|
|
350
450
|
</details>
|
|
351
451
|
|
|
@@ -354,7 +454,7 @@ $ python -m argus.bench --target all
|
|
|
354
454
|
|
|
355
455
|
**BuggyTasks** (`:5555`) — 22 mechanical bugs in a task app: console errors, dead links, fake delete (UI says "deleted!" but data persists on refresh), auth bypass, NaN dates, off-by-one counts, race conditions. The "scripted E2E could find these" tier.
|
|
356
456
|
|
|
357
|
-
**DarkShop** (`:5556`) — 12
|
|
457
|
+
**DarkShop** (`:5556`) — 12 seeded runtime and visual scenarios in a polished-looking store: hardcoded "Only 3 left!" scarcity, `-50%` badges where sale price equals original, a "free shipping over $50" banner contradicted by a flat `$5` at checkout, inverted visual hierarchy ("Add to Cart" demoted under a prominent "Subscribe"), cross-page state drift (rename sticks on /account, navbar greeting doesn't). These fixtures exercise comparisons between visible claims and observed behavior; they are not a benchmark against static analysis tools.
|
|
358
458
|
|
|
359
459
|
```bash
|
|
360
460
|
python test-site/app.py # BuggyTasks :5555
|
|
@@ -379,6 +479,7 @@ uvx --from argus-testing argus-mcp --tool-profile full --list-tools
|
|
|
379
479
|
| Profile | Public tools | Intended use |
|
|
380
480
|
|---------|-------------:|--------------|
|
|
381
481
|
| `core` | 30 | Primary browser QA workflow; the default. |
|
|
482
|
+
| `agent` | 19 | Focused browser journeys, concise descriptions and batch coverage; pair with compact responses when the host supports structured results. |
|
|
382
483
|
| `screen` | 14 | Focused native macOS testing through Accessibility and screenshots. |
|
|
383
484
|
| `full` | 77 | Everything in core and screen, plus specialist browser, state, network, coordinate, and crawl controls. |
|
|
384
485
|
|
|
@@ -400,11 +501,11 @@ uvx --from argus-testing argus-mcp --tool-profile full --list-tools
|
|
|
400
501
|
| `screenshot` · `screenshot_diff` | Capture viewport, full-page, or element evidence and produce a red-tint pixel-diff overlay. |
|
|
401
502
|
| `get_errors` | Drain correlated console errors and HTTP 4xx/5xx events captured since the previous read. |
|
|
402
503
|
| `capsule_save` · `capsule_restore` | Save and restore a named authenticated or seeded browser state, with an optional liveness check. |
|
|
403
|
-
| `verify_persistence` | Force a fresh load and
|
|
504
|
+
| `verify_persistence` | Force a fresh load and return target presence plus the page observation. This checks persistence, not semantic correctness. |
|
|
404
505
|
| `test_action` · `test_form` | Perform a description-keyed action or form submission and return the resulting state diff in one round trip. |
|
|
405
506
|
| `check_links` · `check_performance` | Probe current-page internal links and expose raw browser performance metrics without auto-certifying generic audit findings. |
|
|
406
507
|
| `regression_check` | Re-test journaled findings for the current origin without requiring another discovery pass. |
|
|
407
|
-
| `record_bug` · `record_observation` | Record a
|
|
508
|
+
| `record_bug` · `record_observation` | Record a finding with evidence and its recheck outcome, or keep a qualitative review note separate. Inspect the receipt rather than treating recording as confirmation. |
|
|
408
509
|
| `end_session` | Close the active session and emit HTML, JSON, JUnit, and SARIF reports. |
|
|
409
510
|
|
|
410
511
|
</details>
|
|
@@ -491,7 +592,11 @@ necessary and the host is authorized to receive credentials.
|
|
|
491
592
|
<details>
|
|
492
593
|
<summary><b>Trust the agent, don't simulate intelligence</b></summary>
|
|
493
594
|
|
|
494
|
-
|
|
595
|
+
The host agent makes semantic judgments. Argus supplies bounded observations,
|
|
596
|
+
deterministic checks, evidence-linked coverage and explicit uncertainty.
|
|
597
|
+
Guidance suggests applicable probes; it never silently executes them or treats
|
|
598
|
+
a heuristic as a confirmed defect. Browser events are captured as observations,
|
|
599
|
+
not proof that the user-facing behavior is wrong.
|
|
495
600
|
|
|
496
601
|
</details>
|
|
497
602
|
|
|
@@ -528,7 +633,8 @@ argus/
|
|
|
528
633
|
├── reporter.py # atomic report orchestration and exporter registry
|
|
529
634
|
├── resolver.py # description → element (web + screen)
|
|
530
635
|
├── cli.py # autonomous exploration and shared regression entry point
|
|
531
|
-
|
|
636
|
+
├── evaluation/ # paired schemas, isolated fixtures, opt-in host runner, adjudication
|
|
637
|
+
└── bench/ # deterministic capability + legacy heuristic harness
|
|
532
638
|
test-site/ # BuggyTasks (22 mechanical bugs)
|
|
533
639
|
human-eye-fixture/ # DarkShop (12 human-eye bugs)
|
|
534
640
|
```
|
|
@@ -51,6 +51,11 @@ inputs require approved directories in `ARGUS_FILE_ROOTS`; see [Security](SECURI
|
|
|
51
51
|
model service.
|
|
52
52
|
- Installing or upgrading through PyPI, `pip`, or `uvx` contacts those package
|
|
53
53
|
distribution services under their own policies.
|
|
54
|
+
- The optional `argus.evaluation.codex_runner` sends synthetic testing prompts
|
|
55
|
+
and tool evidence to the operator's ChatGPT-backed Codex session only after
|
|
56
|
+
explicit subscription-usage opt-in. It stores local answers, usage, tool traces
|
|
57
|
+
and fixture oracle logs, not model reasoning. Raw traces can contain local
|
|
58
|
+
paths and should remain private. Normal MCP operation never starts this runner.
|
|
54
59
|
|
|
55
60
|
Argus does not intentionally transmit test artifacts to the project maintainer.
|
|
56
61
|
Review reports and screenshots before sharing them because they can contain
|