argus-testing 0.8.0__tar.gz → 0.9.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {argus_testing-0.8.0 → argus_testing-0.9.0}/AGENTS.md +10 -1
- {argus_testing-0.8.0 → argus_testing-0.9.0}/CHANGELOG.md +17 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/PKG-INFO +46 -16
- {argus_testing-0.8.0 → argus_testing-0.9.0}/README.md +45 -15
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/adapters/monitoring.py +8 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/application/manager.py +8 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/browser.py +10 -0
- argus_testing-0.9.0/argus/entrypoints/mcp/overhead.py +40 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/entrypoints/mcp/results.py +21 -3
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/entrypoints/mcp/server.py +33 -5
- argus_testing-0.9.0/argus/entrypoints/mcp/snapshots.py +77 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/evaluation/gateway.py +8 -1
- argus_testing-0.9.0/argus/evaluation/overhead.py +121 -0
- argus_testing-0.9.0/argus/install_check.py +68 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/mcp_server.py +1 -1
- argus_testing-0.9.0/argus/measurement.py +61 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/reporter.py +2 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/settings.py +2 -2
- {argus_testing-0.8.0 → argus_testing-0.9.0}/docs/agent-protocol.md +48 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/docs/architecture.md +13 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/docs/evaluation.md +41 -0
- argus_testing-0.9.0/examples/response-overhead.json +170 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/llms-install.md +10 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/pyproject.toml +1 -1
- {argus_testing-0.8.0 → argus_testing-0.9.0}/server.json +2 -2
- {argus_testing-0.8.0 → argus_testing-0.9.0}/tests/test_monitoring.py +24 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/tests/test_release_metadata.py +9 -0
- argus_testing-0.9.0/tests/test_response_efficiency.py +235 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/tests/test_stdio_protocol.py +41 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/.gitignore +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/CONTRIBUTING.md +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/LICENSE +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/PRIVACY.md +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/SECURITY.md +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/__init__.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/_compat.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/access.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/adapters/__init__.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/adapters/assets/__init__.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/adapters/assets/dashboard.css +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/adapters/assets/dashboard.html +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/adapters/assets/dashboard.js +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/adapters/journal.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/adapters/logging.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/adapters/reporting/__init__.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/adapters/reporting/common.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/adapters/reporting/html.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/adapters/reporting/json.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/adapters/reporting/junit.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/adapters/reporting/privacy.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/adapters/reporting/sarif.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/adapters/scripts/__init__.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/adapters/scripts/check_layout_js.js +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/adapters/scripts/extract_elements_js.js +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/adapters/scripts/extract_page_content_js.js +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/adapters/scripts/extract_performance_js.js +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/adapters/scripts/find_visible_element_js.js +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/adapters/scripts/inspect_element_js.js +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/application/__init__.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/application/actions.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/application/capsules.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/application/capture.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/application/context.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/application/coverage.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/application/events.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/application/evidence.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/application/expectations.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/application/findings.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/application/finish.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/application/guidance.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/application/inspection.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/application/journeys.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/application/lifecycle.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/application/navigation.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/application/network_storage.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/application/observation.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/application/operations.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/application/payloads.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/application/probes.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/application/progress.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/application/regression.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/application/screen.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/application/screenshots.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/application/session.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/application/telemetry.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/application/text.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/application/verification.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/application/verifiers.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/bench/__init__.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/bench/__main__.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/bench/agent_runner.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/bench/runner.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/bench/scenarios_buggytasks.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/bench/scenarios_darkshop.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/bootstrap.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/cleanup.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/cli.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/config.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/detector.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/differ.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/domain/__init__.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/domain/contracts.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/domain/detector.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/domain/models.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/domain/replay.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/domain/verification.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/entrypoints/__init__.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/entrypoints/mcp/__init__.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/entrypoints/mcp/catalog.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/evaluation/__init__.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/evaluation/__main__.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/evaluation/adjudication.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/evaluation/agent_fixture.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/evaluation/browser_baseline.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/evaluation/codex_runner.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/evaluation/fixture.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/evaluation/models.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/evaluation/scoring.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/evaluation/smoke.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/explorer.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/models.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/planner.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/ports.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/resolver.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/runtime.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/screen/__init__.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/screen/backend.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/screen/permissions.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/screen/safety.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/screen/validate.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/security.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/argus/storage.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/docs/decisions.md +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/docs/evaluation-0.8.md +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/docs/implementation-0.7.md +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/examples/agent-evaluation-optimized-evidence.json +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/examples/agent-evaluation-optimized.json +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/examples/agent-evaluation-pilot-evidence.json +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/examples/agent-evaluation-pilot.json +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/examples/focus.yaml +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/examples/mcp-config.json +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/glama.json +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/human-eye-fixture/app.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/scripts/check_distributions.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/test-site/app.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/tests/__init__.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/tests/conftest.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/tests/test_action_targeting.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/tests/test_adjudication.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/tests/test_agent_results.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/tests/test_agent_workflow.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/tests/test_architecture_runtime.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/tests/test_bench_report.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/tests/test_browser_shadow.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/tests/test_codex_evaluation.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/tests/test_cross_stack.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/tests/test_detector_console.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/tests/test_detector_network.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/tests/test_distribution_contents.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/tests/test_evaluation.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/tests/test_expectation_oracle.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/tests/test_final_hardening.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/tests/test_journal_regression.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/tests/test_mcp_contract.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/tests/test_regression_cli.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/tests/test_replay_minimize.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/tests/test_replay_receipt.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/tests/test_replay_trace.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/tests/test_reporter_badge.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/tests/test_reproduction_receipt.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/tests/test_resolver.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/tests/test_runtime_limits.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/tests/test_screen_permissions.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/tests/test_screen_safety.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/tests/test_security_boundaries.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/tests/test_session_lifecycle.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/tests/test_state_capsules.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/tests/test_storage_cleanup.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/tests/test_substrate.py +0 -0
- {argus_testing-0.8.0 → argus_testing-0.9.0}/tests/test_visual_inspection.py +0 -0
|
@@ -27,6 +27,15 @@ argus-mcp --doctor # macOS Screen Recording + Accessibility check
|
|
|
27
27
|
|
|
28
28
|
The default profile is `core`. For structured-result-aware hosts, use
|
|
29
29
|
`--tool-profile agent --response-mode compact` for 19 focused tools.
|
|
30
|
+
|
|
31
|
+
`--response-mode delta` is opt-in for snapshot-aware clients. Only observation
|
|
32
|
+
fields listed in the protocol may be inherited from an exact `base_id`; action
|
|
33
|
+
and verification results are current-call-only. If context or a base is missing,
|
|
34
|
+
call `observe` for full state. Never infer a passing check from unchanged fields.
|
|
35
|
+
Use `python -m argus.install_check` to verify installed versions and stdio without
|
|
36
|
+
a browser or model, and `python -m argus.evaluation.overhead` for deterministic
|
|
37
|
+
byte measurements. Bytes are not model tokens. Monitoring retains numeric
|
|
38
|
+
overhead only; do not add page content or arguments to metrics.
|
|
30
39
|
Use `--tool-profile screen` for native
|
|
31
40
|
macOS review and `--tool-profile full` only when the broader network,
|
|
32
41
|
storage, tabs, crawl, and coordinate surface is needed. The
|
|
@@ -73,7 +82,7 @@ result includes captured stages. See [the protocol](docs/agent-protocol.md).
|
|
|
73
82
|
| `type_into(description, text)` / `select_into(description, value)` | Inputs and dropdowns by description. |
|
|
74
83
|
| `test_action(target, expect=...)` | Click + before/after diff in one call. Pass `expect` to PREDICT the outcome ({"count":{"label":"tasks","delta":1}}, {"gains":"Buy milk"}, {"removes":...}, {"text_present":...}, {"toast":...}, {"url_changed":true}) and Argus reports MATCH / SURPRISE — a surprise is a bug lead. Also shows CROSS-STACK: which requests the click fired (methods/statuses) and a CHECK nudge when a message appeared without a matching write. |
|
|
75
84
|
| `verify_persistence(expect, target_text, after_url)` | Forces a fresh GET; returns target presence and the page observation. Check persistence once, repeat only to resolve uncertainty. A matching symptom is not proof of a semantic defect. |
|
|
76
|
-
| `capsule_save(name, liveness_marker)` / `capsule_restore(name)` | Snapshot
|
|
85
|
+
| `capsule_save(name, liveness_marker)` / `capsule_restore(name)` | Snapshot cookies and page storage after establishing state through the UI. Restore replaces the captured client state and checks a saved liveness marker when provided; without one, validity remains unverified. It does not reset backend data or guarantee identical application state across journeys. |
|
|
77
86
|
| `regression_check()` | Re-test the findings journaled in prior runs against the CURRENT build: STILL-PRESENT / NO-LONGER-REPRODUCES / INCONCLUSIVE. "Did my fix land, did anything come back?" |
|
|
78
87
|
| `inspect_element(description)` | Computed styles + ARIA + outerHTML + truncation flag for interactive or visible non-interactive content. |
|
|
79
88
|
| `check_layout()` | Bounded viewport signals for horizontal overflow, clipped text, small targets, and fixed/sticky overlays. |
|
|
@@ -1,5 +1,22 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.9.0 - 2026-10-25
|
|
4
|
+
|
|
5
|
+
- Report the Argus product version in the MCP handshake instead of the SDK
|
|
6
|
+
version. Add a no-browser installed-package stdio check and run it against
|
|
7
|
+
fresh release wheels outside the source checkout.
|
|
8
|
+
- Remove redundant output-schema display titles without changing validation
|
|
9
|
+
constraints or tool display titles.
|
|
10
|
+
- Add opt-in delta observations with explicit base IDs, field removals, full
|
|
11
|
+
resynchronization, lifecycle invalidation and bounded redacted retention.
|
|
12
|
+
Preserve errors, current-call verdicts, image blocks and compatibility defaults.
|
|
13
|
+
- Measure response bytes and instrumented browser waits, observation, screenshots,
|
|
14
|
+
report export and encoding. Expose numeric totals through loopback status and
|
|
15
|
+
Prometheus; keep page content and input values out of monitoring.
|
|
16
|
+
- Add reproducible, lossless encoding comparisons with navigation controls and
|
|
17
|
+
gateway discovery/response measurements. Do not equate byte savings with model
|
|
18
|
+
token savings or claim a new model effectiveness evaluation.
|
|
19
|
+
|
|
3
20
|
## 0.8.0 - 2026-10-25
|
|
4
21
|
|
|
5
22
|
- Add an optional 19-tool agent profile with concise discovery descriptions,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: argus-testing
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.9.0
|
|
4
4
|
Summary: Agent-first testing MCP with evidence-backed coverage, persistence checks and reproduction receipts
|
|
5
5
|
Project-URL: Homepage, https://yichenwu.dev/projects/argus
|
|
6
6
|
Project-URL: Documentation, https://github.com/chriswu727/argus#readme
|
|
@@ -67,6 +67,8 @@ Argus is an [MCP](https://modelcontextprotocol.io/) server that gives your codin
|
|
|
67
67
|
[](#benchmarks)
|
|
68
68
|
[](LICENSE)
|
|
69
69
|
|
|
70
|
+
[](https://mcprush.com/to/argus-testing-autonomous-qa-mcp?ref=badge)
|
|
71
|
+
|
|
70
72
|
[Product page](https://yichenwu.dev/projects/argus) · [Quick start](#quick-start) · [Why Argus](#why-argus-is-different) · [Compared](#how-it-compares) · [Tools](#tool-surface) · [Benchmarks](#benchmarks)
|
|
71
73
|
|
|
72
74
|
</div>
|
|
@@ -94,10 +96,14 @@ the phase summary in the final result. See the [agent protocol](docs/agent-proto
|
|
|
94
96
|
- **Conservative checks:** missing or malformed evidence stays unknown, not a
|
|
95
97
|
guessed pass or a fabricated bug. Findings link to stable session references.
|
|
96
98
|
- **Focused agent mode:** `argus-mcp --tool-profile agent --response-mode compact`
|
|
97
|
-
exposes 19 common testing tools with concise discovery descriptions
|
|
99
|
+
exposes 19 common testing tools with concise discovery descriptions, preserves structured evidence,
|
|
98
100
|
and avoids repeating migrated payloads as prose. Compatible mode remains the default.
|
|
99
101
|
- **Visible progress:** real phases, goal counts and check counts over MCP;
|
|
100
102
|
`--monitor-port 8765` adds the local live dashboard without exposing goal text.
|
|
103
|
+
- **Measured response overhead (0.9):** smaller output schemas, numeric payload
|
|
104
|
+
and component-timing counters, and opt-in `--response-mode delta` for clients
|
|
105
|
+
that retain observation snapshots. Missing bases require a full `observe`;
|
|
106
|
+
verification outcomes and image blocks are never replaced by cached verdicts.
|
|
101
107
|
|
|
102
108
|
Does installing Argus improve an agent? Measure it with the
|
|
103
109
|
[paired evaluation harness](docs/evaluation.md): the same model, task, fixture,
|
|
@@ -111,6 +117,11 @@ declared-goal coverage. Argus used more time and tokens. On four repeated pilot
|
|
|
111
117
|
cases, the targeted update reduced Argus's median calls from 17 to 9; this small
|
|
112
118
|
synthetic sample does not establish general superiority.
|
|
113
119
|
|
|
120
|
+
The no-model [response encoding benchmark](docs/evaluation.md#response-overhead)
|
|
121
|
+
compares identical synthetic evidence, including navigation invalidation. Delta
|
|
122
|
+
mode reduces aggregate result JSON by 47.09% versus compact mode across those
|
|
123
|
+
24 responses; this measures bytes, **not agent token usage or testing quality**.
|
|
124
|
+
|
|
114
125
|
<details>
|
|
115
126
|
<summary><b>Live monitoring from a real local fixture run</b></summary>
|
|
116
127
|
|
|
@@ -148,10 +159,10 @@ flowchart LR
|
|
|
148
159
|
A(["observe"]) --> B{"looks wrong?"}
|
|
149
160
|
B -->|not sure| C["act: click · type · resize · verify"]
|
|
150
161
|
C --> A
|
|
151
|
-
B -->|
|
|
152
|
-
D
|
|
153
|
-
|
|
154
|
-
|
|
162
|
+
B -->|suspected defect| D["verify_persistence — inspect a fresh load"]
|
|
163
|
+
D --> E["record_bug with an independent recheck"]
|
|
164
|
+
E --> F(["reproduced / not reproduced / inconclusive"])
|
|
165
|
+
F --> G[["report: HTML · JSON · JUnit · SARIF"]]
|
|
155
166
|
```
|
|
156
167
|
|
|
157
168
|
The agent supplies the judgment. Argus supplies concise QA guidance, a description-keyed tool surface (`click_what("Login button")`, not `click(7)`), a goal coverage ledger, and a **reproduction-receipt engine** that distinguishes observed, reproduced and inconclusive symptoms.
|
|
@@ -212,7 +223,7 @@ The button adds Argus to Cursor; run the Chromium installation command above onc
|
|
|
212
223
|
|
|
213
224
|
The default `core` profile exposes the primary web-testing workflow without flooding the host with every specialist tool. Use `uvx --from argus-testing argus-mcp --list-tools` to inspect the selected profile, `--tool-profile screen` for native macOS testing, or `--tool-profile full` for the entire advanced surface. `ARGUS_TOOL_PROFILE` provides the same setting through the environment.
|
|
214
225
|
|
|
215
|
-
For agents that read structured MCP results, use the focused
|
|
226
|
+
For agents that read structured MCP results, use the focused configuration:
|
|
216
227
|
|
|
217
228
|
```bash
|
|
218
229
|
uvx --from argus-testing argus-mcp --tool-profile agent --response-mode compact --monitor-port 8765
|
|
@@ -222,6 +233,21 @@ Read returned observations before issuing another `observe`. Batch related goals
|
|
|
222
233
|
with `coverage_update(goal=["edit profile", "reload profile"], status="in_progress")`;
|
|
223
234
|
finish them with observed evidence, not an assumed pass. The monitor stays local.
|
|
224
235
|
|
|
236
|
+
`--response-mode delta` is an additional opt-in for clients that can retain and
|
|
237
|
+
merge snapshot IDs. Keep `compact` if the host cannot reliably resolve bases.
|
|
238
|
+
See the [snapshot contract](docs/agent-protocol.md#incremental-observations).
|
|
239
|
+
|
|
240
|
+
To check installation, product version, stdio handshake, discovery and structured
|
|
241
|
+
errors without opening a browser or calling a model:
|
|
242
|
+
|
|
243
|
+
```bash
|
|
244
|
+
uvx --from argus-testing python -m argus.install_check
|
|
245
|
+
```
|
|
246
|
+
|
|
247
|
+
Editable installs must be reinstalled after a version change so their package
|
|
248
|
+
metadata matches the source. MCP `serverInfo.version` now reports Argus's version,
|
|
249
|
+
not the MCP SDK version.
|
|
250
|
+
|
|
225
251
|
Then just ask, in your agent session:
|
|
226
252
|
|
|
227
253
|
> **"Test my app at http://localhost:3000 — find real bugs."**
|
|
@@ -313,9 +339,9 @@ selected URL is printed to **stderr**, so stdio MCP messages stay intact.
|
|
|
313
339
|
| `/` | Local live dashboard |
|
|
314
340
|
| `/healthz` | HTTP process liveness |
|
|
315
341
|
| `/readyz` | Runtime initialized and not shutting down; 503 otherwise |
|
|
316
|
-
| `/api/status` | Versioned state snapshot, counts
|
|
342
|
+
| `/api/status` | Versioned state snapshot, counts, capture drops and numeric overhead |
|
|
317
343
|
| `/api/events` | Server-Sent Events, `Last-Event-ID` reconnect and gap notices |
|
|
318
|
-
| `/metrics` | Prometheus
|
|
344
|
+
| `/metrics` | Prometheus operation, response-byte and component-timing counters |
|
|
319
345
|
|
|
320
346
|
The monitor binds only to `127.0.0.1`, rejects unrelated Host/Origin headers,
|
|
321
347
|
and has no action endpoints. It emits no page contents, URLs, typed values,
|
|
@@ -361,16 +387,20 @@ application data; see [Privacy](PRIVACY.md).
|
|
|
361
387
|
|
|
362
388
|
## Why Argus is different
|
|
363
389
|
|
|
364
|
-
**
|
|
390
|
+
**Give exploratory testing a repeatable evidence workflow.** An agent can use
|
|
391
|
+
Argus to investigate journeys beyond prewritten checks, inspect what changed,
|
|
392
|
+
and independently recheck suspected symptoms. The agent still decides whether
|
|
393
|
+
the observed behavior violates the application's requirements; a reproduction
|
|
394
|
+
receipt is supporting evidence, not a guarantee of correctness or zero false positives.
|
|
365
395
|
|
|
366
396
|
| | |
|
|
367
397
|
|---|---|
|
|
368
398
|
| **Agent-directed & black-box** | Give your agent a URL and scope. It explores through Argus without requiring application source access or prewritten test scripts. |
|
|
369
399
|
| **Coverage contract** | Optional natural-language goals, user constraints, discovered pages, and time budget stay visible throughout the session and in the final report. |
|
|
370
|
-
| **Reproduction receipts** |
|
|
371
|
-
| **
|
|
372
|
-
| **Discover → guard** |
|
|
373
|
-
| **Machine-readable** |
|
|
400
|
+
| **Reproduction receipts** | Independent checks distinguish reproduced, not-reproduced and inconclusive symptoms. Reloads preserve authentication and storage by default; replay can repeat writes. |
|
|
401
|
+
| **Runtime evidence** | Inspect misleading success messages, cross-page inconsistencies and persistence failures in the running app. The agent interprets this evidence against the intended behavior. |
|
|
402
|
+
| **Discover → guard** | Add `argus-regression` to CI to recheck eligible journaled findings without model calls. Exit codes distinguish reproduced symptoms, operational failures and inconclusive checks. |
|
|
403
|
+
| **Machine-readable** | JSON, JUnit and SARIF exports support downstream processing. CI gates and code-scanning uploads require integration; URL-based findings do not guarantee source-line PR annotations. |
|
|
374
404
|
|
|
375
405
|
---
|
|
376
406
|
|
|
@@ -424,7 +454,7 @@ not the basis for current product claims. See [methodology](docs/evaluation.md).
|
|
|
424
454
|
|
|
425
455
|
**BuggyTasks** (`:5555`) — 22 mechanical bugs in a task app: console errors, dead links, fake delete (UI says "deleted!" but data persists on refresh), auth bypass, NaN dates, off-by-one counts, race conditions. The "scripted E2E could find these" tier.
|
|
426
456
|
|
|
427
|
-
**DarkShop** (`:5556`) — 12
|
|
457
|
+
**DarkShop** (`:5556`) — 12 seeded runtime and visual scenarios in a polished-looking store: hardcoded "Only 3 left!" scarcity, `-50%` badges where sale price equals original, a "free shipping over $50" banner contradicted by a flat `$5` at checkout, inverted visual hierarchy ("Add to Cart" demoted under a prominent "Subscribe"), cross-page state drift (rename sticks on /account, navbar greeting doesn't). These fixtures exercise comparisons between visible claims and observed behavior; they are not a benchmark against static analysis tools.
|
|
428
458
|
|
|
429
459
|
```bash
|
|
430
460
|
python test-site/app.py # BuggyTasks :5555
|
|
@@ -475,7 +505,7 @@ uvx --from argus-testing argus-mcp --tool-profile full --list-tools
|
|
|
475
505
|
| `test_action` · `test_form` | Perform a description-keyed action or form submission and return the resulting state diff in one round trip. |
|
|
476
506
|
| `check_links` · `check_performance` | Probe current-page internal links and expose raw browser performance metrics without auto-certifying generic audit findings. |
|
|
477
507
|
| `regression_check` | Re-test journaled findings for the current origin without requiring another discovery pass. |
|
|
478
|
-
| `record_bug` · `record_observation` | Record a
|
|
508
|
+
| `record_bug` · `record_observation` | Record a finding with evidence and its recheck outcome, or keep a qualitative review note separate. Inspect the receipt rather than treating recording as confirmation. |
|
|
479
509
|
| `end_session` | Close the active session and emit HTML, JSON, JUnit, and SARIF reports. |
|
|
480
510
|
|
|
481
511
|
</details>
|
|
@@ -17,6 +17,8 @@ Argus is an [MCP](https://modelcontextprotocol.io/) server that gives your codin
|
|
|
17
17
|
[](#benchmarks)
|
|
18
18
|
[](LICENSE)
|
|
19
19
|
|
|
20
|
+
[](https://mcprush.com/to/argus-testing-autonomous-qa-mcp?ref=badge)
|
|
21
|
+
|
|
20
22
|
[Product page](https://yichenwu.dev/projects/argus) · [Quick start](#quick-start) · [Why Argus](#why-argus-is-different) · [Compared](#how-it-compares) · [Tools](#tool-surface) · [Benchmarks](#benchmarks)
|
|
21
23
|
|
|
22
24
|
</div>
|
|
@@ -44,10 +46,14 @@ the phase summary in the final result. See the [agent protocol](docs/agent-proto
|
|
|
44
46
|
- **Conservative checks:** missing or malformed evidence stays unknown, not a
|
|
45
47
|
guessed pass or a fabricated bug. Findings link to stable session references.
|
|
46
48
|
- **Focused agent mode:** `argus-mcp --tool-profile agent --response-mode compact`
|
|
47
|
-
exposes 19 common testing tools with concise discovery descriptions
|
|
49
|
+
exposes 19 common testing tools with concise discovery descriptions, preserves structured evidence,
|
|
48
50
|
and avoids repeating migrated payloads as prose. Compatible mode remains the default.
|
|
49
51
|
- **Visible progress:** real phases, goal counts and check counts over MCP;
|
|
50
52
|
`--monitor-port 8765` adds the local live dashboard without exposing goal text.
|
|
53
|
+
- **Measured response overhead (0.9):** smaller output schemas, numeric payload
|
|
54
|
+
and component-timing counters, and opt-in `--response-mode delta` for clients
|
|
55
|
+
that retain observation snapshots. Missing bases require a full `observe`;
|
|
56
|
+
verification outcomes and image blocks are never replaced by cached verdicts.
|
|
51
57
|
|
|
52
58
|
Does installing Argus improve an agent? Measure it with the
|
|
53
59
|
[paired evaluation harness](docs/evaluation.md): the same model, task, fixture,
|
|
@@ -61,6 +67,11 @@ declared-goal coverage. Argus used more time and tokens. On four repeated pilot
|
|
|
61
67
|
cases, the targeted update reduced Argus's median calls from 17 to 9; this small
|
|
62
68
|
synthetic sample does not establish general superiority.
|
|
63
69
|
|
|
70
|
+
The no-model [response encoding benchmark](docs/evaluation.md#response-overhead)
|
|
71
|
+
compares identical synthetic evidence, including navigation invalidation. Delta
|
|
72
|
+
mode reduces aggregate result JSON by 47.09% versus compact mode across those
|
|
73
|
+
24 responses; this measures bytes, **not agent token usage or testing quality**.
|
|
74
|
+
|
|
64
75
|
<details>
|
|
65
76
|
<summary><b>Live monitoring from a real local fixture run</b></summary>
|
|
66
77
|
|
|
@@ -98,10 +109,10 @@ flowchart LR
|
|
|
98
109
|
A(["observe"]) --> B{"looks wrong?"}
|
|
99
110
|
B -->|not sure| C["act: click · type · resize · verify"]
|
|
100
111
|
C --> A
|
|
101
|
-
B -->|
|
|
102
|
-
D
|
|
103
|
-
|
|
104
|
-
|
|
112
|
+
B -->|suspected defect| D["verify_persistence — inspect a fresh load"]
|
|
113
|
+
D --> E["record_bug with an independent recheck"]
|
|
114
|
+
E --> F(["reproduced / not reproduced / inconclusive"])
|
|
115
|
+
F --> G[["report: HTML · JSON · JUnit · SARIF"]]
|
|
105
116
|
```
|
|
106
117
|
|
|
107
118
|
The agent supplies the judgment. Argus supplies concise QA guidance, a description-keyed tool surface (`click_what("Login button")`, not `click(7)`), a goal coverage ledger, and a **reproduction-receipt engine** that distinguishes observed, reproduced and inconclusive symptoms.
|
|
@@ -162,7 +173,7 @@ The button adds Argus to Cursor; run the Chromium installation command above onc
|
|
|
162
173
|
|
|
163
174
|
The default `core` profile exposes the primary web-testing workflow without flooding the host with every specialist tool. Use `uvx --from argus-testing argus-mcp --list-tools` to inspect the selected profile, `--tool-profile screen` for native macOS testing, or `--tool-profile full` for the entire advanced surface. `ARGUS_TOOL_PROFILE` provides the same setting through the environment.
|
|
164
175
|
|
|
165
|
-
For agents that read structured MCP results, use the focused
|
|
176
|
+
For agents that read structured MCP results, use the focused configuration:
|
|
166
177
|
|
|
167
178
|
```bash
|
|
168
179
|
uvx --from argus-testing argus-mcp --tool-profile agent --response-mode compact --monitor-port 8765
|
|
@@ -172,6 +183,21 @@ Read returned observations before issuing another `observe`. Batch related goals
|
|
|
172
183
|
with `coverage_update(goal=["edit profile", "reload profile"], status="in_progress")`;
|
|
173
184
|
finish them with observed evidence, not an assumed pass. The monitor stays local.
|
|
174
185
|
|
|
186
|
+
`--response-mode delta` is an additional opt-in for clients that can retain and
|
|
187
|
+
merge snapshot IDs. Keep `compact` if the host cannot reliably resolve bases.
|
|
188
|
+
See the [snapshot contract](docs/agent-protocol.md#incremental-observations).
|
|
189
|
+
|
|
190
|
+
To check installation, product version, stdio handshake, discovery and structured
|
|
191
|
+
errors without opening a browser or calling a model:
|
|
192
|
+
|
|
193
|
+
```bash
|
|
194
|
+
uvx --from argus-testing python -m argus.install_check
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
Editable installs must be reinstalled after a version change so their package
|
|
198
|
+
metadata matches the source. MCP `serverInfo.version` now reports Argus's version,
|
|
199
|
+
not the MCP SDK version.
|
|
200
|
+
|
|
175
201
|
Then just ask, in your agent session:
|
|
176
202
|
|
|
177
203
|
> **"Test my app at http://localhost:3000 — find real bugs."**
|
|
@@ -263,9 +289,9 @@ selected URL is printed to **stderr**, so stdio MCP messages stay intact.
|
|
|
263
289
|
| `/` | Local live dashboard |
|
|
264
290
|
| `/healthz` | HTTP process liveness |
|
|
265
291
|
| `/readyz` | Runtime initialized and not shutting down; 503 otherwise |
|
|
266
|
-
| `/api/status` | Versioned state snapshot, counts
|
|
292
|
+
| `/api/status` | Versioned state snapshot, counts, capture drops and numeric overhead |
|
|
267
293
|
| `/api/events` | Server-Sent Events, `Last-Event-ID` reconnect and gap notices |
|
|
268
|
-
| `/metrics` | Prometheus
|
|
294
|
+
| `/metrics` | Prometheus operation, response-byte and component-timing counters |
|
|
269
295
|
|
|
270
296
|
The monitor binds only to `127.0.0.1`, rejects unrelated Host/Origin headers,
|
|
271
297
|
and has no action endpoints. It emits no page contents, URLs, typed values,
|
|
@@ -311,16 +337,20 @@ application data; see [Privacy](PRIVACY.md).
|
|
|
311
337
|
|
|
312
338
|
## Why Argus is different
|
|
313
339
|
|
|
314
|
-
**
|
|
340
|
+
**Give exploratory testing a repeatable evidence workflow.** An agent can use
|
|
341
|
+
Argus to investigate journeys beyond prewritten checks, inspect what changed,
|
|
342
|
+
and independently recheck suspected symptoms. The agent still decides whether
|
|
343
|
+
the observed behavior violates the application's requirements; a reproduction
|
|
344
|
+
receipt is supporting evidence, not a guarantee of correctness or zero false positives.
|
|
315
345
|
|
|
316
346
|
| | |
|
|
317
347
|
|---|---|
|
|
318
348
|
| **Agent-directed & black-box** | Give your agent a URL and scope. It explores through Argus without requiring application source access or prewritten test scripts. |
|
|
319
349
|
| **Coverage contract** | Optional natural-language goals, user constraints, discovered pages, and time budget stay visible throughout the session and in the final report. |
|
|
320
|
-
| **Reproduction receipts** |
|
|
321
|
-
| **
|
|
322
|
-
| **Discover → guard** |
|
|
323
|
-
| **Machine-readable** |
|
|
350
|
+
| **Reproduction receipts** | Independent checks distinguish reproduced, not-reproduced and inconclusive symptoms. Reloads preserve authentication and storage by default; replay can repeat writes. |
|
|
351
|
+
| **Runtime evidence** | Inspect misleading success messages, cross-page inconsistencies and persistence failures in the running app. The agent interprets this evidence against the intended behavior. |
|
|
352
|
+
| **Discover → guard** | Add `argus-regression` to CI to recheck eligible journaled findings without model calls. Exit codes distinguish reproduced symptoms, operational failures and inconclusive checks. |
|
|
353
|
+
| **Machine-readable** | JSON, JUnit and SARIF exports support downstream processing. CI gates and code-scanning uploads require integration; URL-based findings do not guarantee source-line PR annotations. |
|
|
324
354
|
|
|
325
355
|
---
|
|
326
356
|
|
|
@@ -374,7 +404,7 @@ not the basis for current product claims. See [methodology](docs/evaluation.md).
|
|
|
374
404
|
|
|
375
405
|
**BuggyTasks** (`:5555`) — 22 mechanical bugs in a task app: console errors, dead links, fake delete (UI says "deleted!" but data persists on refresh), auth bypass, NaN dates, off-by-one counts, race conditions. The "scripted E2E could find these" tier.
|
|
376
406
|
|
|
377
|
-
**DarkShop** (`:5556`) — 12
|
|
407
|
+
**DarkShop** (`:5556`) — 12 seeded runtime and visual scenarios in a polished-looking store: hardcoded "Only 3 left!" scarcity, `-50%` badges where sale price equals original, a "free shipping over $50" banner contradicted by a flat `$5` at checkout, inverted visual hierarchy ("Add to Cart" demoted under a prominent "Subscribe"), cross-page state drift (rename sticks on /account, navbar greeting doesn't). These fixtures exercise comparisons between visible claims and observed behavior; they are not a benchmark against static analysis tools.
|
|
378
408
|
|
|
379
409
|
```bash
|
|
380
410
|
python test-site/app.py # BuggyTasks :5555
|
|
@@ -425,7 +455,7 @@ uvx --from argus-testing argus-mcp --tool-profile full --list-tools
|
|
|
425
455
|
| `test_action` · `test_form` | Perform a description-keyed action or form submission and return the resulting state diff in one round trip. |
|
|
426
456
|
| `check_links` · `check_performance` | Probe current-page internal links and expose raw browser performance metrics without auto-certifying generic audit findings. |
|
|
427
457
|
| `regression_check` | Re-test journaled findings for the current origin without requiring another discovery pass. |
|
|
428
|
-
| `record_bug` · `record_observation` | Record a
|
|
458
|
+
| `record_bug` · `record_observation` | Record a finding with evidence and its recheck outcome, or keep a qualitative review note separate. Inspect the receipt rather than treating recording as confirmation. |
|
|
429
459
|
| `end_session` | Close the active session and emit HTML, JSON, JUnit, and SARIF reports. |
|
|
430
460
|
|
|
431
461
|
</details>
|
|
@@ -132,6 +132,14 @@ class Monitor:
|
|
|
132
132
|
f"argus_operation_duration_seconds_count{{operation={label}}} {count}",
|
|
133
133
|
]
|
|
134
134
|
)
|
|
135
|
+
for name, value in sorted(runtime.response_totals.items()):
|
|
136
|
+
lines.append(f"argus_response_total{{measurement={json.dumps(name)}}} {value}")
|
|
137
|
+
for name, value in sorted(runtime.component_seconds.items()):
|
|
138
|
+
label = json.dumps(name)
|
|
139
|
+
lines.append(f"argus_component_seconds_sum{{component={label}}} {value}")
|
|
140
|
+
lines.append(
|
|
141
|
+
f"argus_component_seconds_count{{component={label}}} {runtime.component_calls[name]}"
|
|
142
|
+
)
|
|
135
143
|
return PlainTextResponse("\n".join(lines) + "\n", media_type="text/plain; version=0.0.4")
|
|
136
144
|
|
|
137
145
|
async def asset(self, request: Request) -> Response:
|
|
@@ -48,6 +48,9 @@ class SessionManager:
|
|
|
48
48
|
self.calls: Counter[str] = Counter()
|
|
49
49
|
self.failures: Counter[str] = Counter()
|
|
50
50
|
self.durations: dict[str, float] = defaultdict(float)
|
|
51
|
+
self.response_totals: Counter[str] = Counter()
|
|
52
|
+
self.component_seconds: Counter[str] = Counter()
|
|
53
|
+
self.component_calls: Counter[str] = Counter()
|
|
51
54
|
self.current_operation: str | None = None
|
|
52
55
|
self.operation_started: float | None = None
|
|
53
56
|
self.started = time.monotonic()
|
|
@@ -262,6 +265,11 @@ class SessionManager:
|
|
|
262
265
|
"screenshots": len(session.screenshots),
|
|
263
266
|
"capture_drops": drops,
|
|
264
267
|
"events_dropped": self.events.dropped,
|
|
268
|
+
"overhead": {
|
|
269
|
+
"response_totals": dict(self.response_totals),
|
|
270
|
+
"component_seconds": dict(self.component_seconds),
|
|
271
|
+
"component_calls": dict(self.component_calls),
|
|
272
|
+
},
|
|
265
273
|
}
|
|
266
274
|
|
|
267
275
|
async def shutdown(self) -> None:
|
|
@@ -8,6 +8,7 @@ from playwright.async_api import Browser, BrowserContext, Page, async_playwright
|
|
|
8
8
|
|
|
9
9
|
from argus.access import AccessPolicy
|
|
10
10
|
from argus.domain.contracts import OperationError
|
|
11
|
+
from argus.measurement import timed
|
|
11
12
|
from argus.security import _redact as _redact
|
|
12
13
|
from argus.security import _redact_headers as _redact_headers
|
|
13
14
|
from argus.security import redact_body
|
|
@@ -205,6 +206,7 @@ class BrowserDriver:
|
|
|
205
206
|
async def click_resolved(self, index: int, elements: list[InteractiveElement]) -> None:
|
|
206
207
|
await self._locator(index, elements).click(timeout=5000)
|
|
207
208
|
|
|
209
|
+
@timed("browser_wait")
|
|
208
210
|
async def settle(self) -> None:
|
|
209
211
|
await self._page.wait_for_load_state("networkidle", timeout=10_000)
|
|
210
212
|
|
|
@@ -807,6 +809,7 @@ class BrowserDriver:
|
|
|
807
809
|
|
|
808
810
|
# -- waits --
|
|
809
811
|
|
|
812
|
+
@timed("browser_wait")
|
|
810
813
|
async def wait_for_text(self, text: str, timeout_s: float = 10.0) -> bool:
|
|
811
814
|
"""Poll until the given text appears anywhere in the page body."""
|
|
812
815
|
try:
|
|
@@ -819,6 +822,7 @@ class BrowserDriver:
|
|
|
819
822
|
except Exception:
|
|
820
823
|
return False
|
|
821
824
|
|
|
825
|
+
@timed("browser_wait")
|
|
822
826
|
async def wait_for_request(
|
|
823
827
|
self,
|
|
824
828
|
url_substring: str,
|
|
@@ -854,6 +858,7 @@ class BrowserDriver:
|
|
|
854
858
|
|
|
855
859
|
# -- navigation --
|
|
856
860
|
|
|
861
|
+
@timed("browser_wait")
|
|
857
862
|
async def _settle_network(self, page=None, timeout: int = 6_000):
|
|
858
863
|
"""Best-effort wait for network to go idle. A never-resolving fetch (a
|
|
859
864
|
stuck 'Loading…' spinner) means networkidle NEVER fires, so this must
|
|
@@ -885,6 +890,7 @@ class BrowserDriver:
|
|
|
885
890
|
|
|
886
891
|
# -- state extraction --
|
|
887
892
|
|
|
893
|
+
@timed("observation")
|
|
888
894
|
async def get_state(self, page=None) -> PageState:
|
|
889
895
|
page = page or self._page
|
|
890
896
|
if page is None:
|
|
@@ -1222,6 +1228,7 @@ class BrowserDriver:
|
|
|
1222
1228
|
except Exception:
|
|
1223
1229
|
pass
|
|
1224
1230
|
|
|
1231
|
+
@timed("browser_wait")
|
|
1225
1232
|
async def _settle_dom(self, cap_ms: int = 1500, quiet_ms: int = 300) -> None:
|
|
1226
1233
|
"""Wait until the DOM stops mutating for `quiet_ms` (capped at `cap_ms`).
|
|
1227
1234
|
networkidle doesn't cover pure-JS DOM updates that fire after a click —
|
|
@@ -1245,6 +1252,7 @@ class BrowserDriver:
|
|
|
1245
1252
|
except Exception:
|
|
1246
1253
|
pass # navigation destroyed the context, eval blocked, etc. — best-effort
|
|
1247
1254
|
|
|
1255
|
+
@timed("browser_wait")
|
|
1248
1256
|
async def _settle_visual(self, cap_ms: int = 1200) -> None:
|
|
1249
1257
|
try:
|
|
1250
1258
|
await self._page.evaluate(
|
|
@@ -1652,12 +1660,14 @@ class BrowserDriver:
|
|
|
1652
1660
|
await self._page.evaluate("(d) => window.scrollBy(0, d)", int(px))
|
|
1653
1661
|
await asyncio.sleep(0.4)
|
|
1654
1662
|
|
|
1663
|
+
@timed("screenshot")
|
|
1655
1664
|
async def screenshot(self, path: str, full_page: bool = False) -> str:
|
|
1656
1665
|
Path(path).parent.mkdir(parents=True, exist_ok=True)
|
|
1657
1666
|
await self._settle_visual()
|
|
1658
1667
|
await self._page.screenshot(path=path, full_page=full_page)
|
|
1659
1668
|
return path
|
|
1660
1669
|
|
|
1670
|
+
@timed("screenshot")
|
|
1661
1671
|
async def element_screenshot(self, selector: str, path: str) -> Optional[str]:
|
|
1662
1672
|
"""Capture just the bounds of one element (Playwright crops natively)."""
|
|
1663
1673
|
Path(path).parent.mkdir(parents=True, exist_ok=True)
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
|
|
5
|
+
from mcp.types import CallToolResult, Tool
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def json_bytes(value) -> int:
|
|
9
|
+
return len(json.dumps(value, ensure_ascii=False, separators=(",", ":")).encode("utf-8"))
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def response_bytes(result: CallToolResult) -> dict[str, int]:
|
|
13
|
+
return {
|
|
14
|
+
"result_json_bytes": len(result.model_dump_json(by_alias=True, exclude_none=True).encode()),
|
|
15
|
+
"text_utf8_bytes": sum(
|
|
16
|
+
len(block.text.encode()) for block in result.content if block.type == "text"
|
|
17
|
+
),
|
|
18
|
+
"structured_json_bytes": json_bytes(result.structuredContent)
|
|
19
|
+
if result.structuredContent is not None
|
|
20
|
+
else 0,
|
|
21
|
+
"image_base64_bytes": sum(
|
|
22
|
+
len(block.data.encode()) for block in result.content if block.type == "image"
|
|
23
|
+
),
|
|
24
|
+
"images": sum(block.type == "image" for block in result.content),
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def discovery_bytes(tools: list[Tool], instructions: str = "") -> dict[str, int]:
|
|
29
|
+
return {
|
|
30
|
+
"tools": len(tools),
|
|
31
|
+
"tools_json_bytes": json_bytes(
|
|
32
|
+
[tool.model_dump(mode="json", by_alias=True, exclude_none=True) for tool in tools]
|
|
33
|
+
),
|
|
34
|
+
"instructions_utf8_bytes": len(instructions.encode()),
|
|
35
|
+
"description_utf8_bytes": sum(len((tool.description or "").encode()) for tool in tools),
|
|
36
|
+
"input_schema_json_bytes": sum(json_bytes(tool.inputSchema) for tool in tools),
|
|
37
|
+
"output_schema_json_bytes": sum(
|
|
38
|
+
json_bytes(tool.outputSchema) for tool in tools if tool.outputSchema
|
|
39
|
+
),
|
|
40
|
+
}
|
|
@@ -4,14 +4,25 @@ from typing import Any, Literal
|
|
|
4
4
|
|
|
5
5
|
from mcp.server.fastmcp.utilities.types import Image
|
|
6
6
|
from mcp.types import CallToolResult, TextContent
|
|
7
|
-
from pydantic import BaseModel, Field
|
|
7
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
8
8
|
|
|
9
9
|
from argus.application.progress import Progress
|
|
10
10
|
from argus.domain.contracts import ImageArtifact, OperationResult, ToolText
|
|
11
|
+
from argus.measurement import timed
|
|
11
12
|
from argus.security import redacted_copy
|
|
12
13
|
|
|
14
|
+
from .snapshots import ObservationSnapshots
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _omit_schema_titles(schema: dict) -> None:
|
|
18
|
+
schema.pop("title", None)
|
|
19
|
+
for field_schema in schema.get("properties", {}).values():
|
|
20
|
+
field_schema.pop("title", None)
|
|
21
|
+
|
|
13
22
|
|
|
14
23
|
class AgentResult(BaseModel):
|
|
24
|
+
model_config = ConfigDict(json_schema_extra=_omit_schema_titles)
|
|
25
|
+
|
|
15
26
|
schema_version: Literal[1] = 1
|
|
16
27
|
operation: str
|
|
17
28
|
status: Literal["completed", "error"]
|
|
@@ -36,8 +47,13 @@ _RECOVERY = {
|
|
|
36
47
|
}
|
|
37
48
|
|
|
38
49
|
|
|
50
|
+
@timed("response_encode")
|
|
39
51
|
def to_mcp_result(
|
|
40
|
-
name: str,
|
|
52
|
+
name: str,
|
|
53
|
+
result: OperationResult,
|
|
54
|
+
progress: Progress,
|
|
55
|
+
mode: str = "compatible",
|
|
56
|
+
snapshots: ObservationSnapshots | None = None,
|
|
41
57
|
) -> CallToolResult:
|
|
42
58
|
value = result.value
|
|
43
59
|
code = result.error.value if result.error else None
|
|
@@ -56,13 +72,15 @@ def to_mcp_result(
|
|
|
56
72
|
]
|
|
57
73
|
if artifacts:
|
|
58
74
|
data["artifacts"] = artifacts
|
|
59
|
-
if mode
|
|
75
|
+
if mode in {"compact", "delta"} and not code and isinstance(value, ToolText) and data:
|
|
60
76
|
content = [
|
|
61
77
|
TextContent(
|
|
62
78
|
type="text",
|
|
63
79
|
text=f"{name}: completed. Read structuredContent.data for evidence; completion is not a test verdict.",
|
|
64
80
|
)
|
|
65
81
|
]
|
|
82
|
+
if mode == "delta" and snapshots is not None:
|
|
83
|
+
data = snapshots.encode(data, name, result.run_id, error=bool(code))
|
|
66
84
|
envelope = AgentResult(
|
|
67
85
|
operation=name,
|
|
68
86
|
status="error" if code else "completed",
|
|
@@ -8,10 +8,12 @@ from mcp.server.fastmcp.exceptions import ToolError
|
|
|
8
8
|
from mcp.server.fastmcp.utilities.types import Image
|
|
9
9
|
from mcp.types import ToolAnnotations
|
|
10
10
|
|
|
11
|
+
from argus import __version__
|
|
11
12
|
from argus.application.manager import SessionManager
|
|
12
13
|
from argus.application.operations import OPERATIONS
|
|
13
14
|
from argus.application.progress import progress_scope
|
|
14
15
|
from argus.domain.contracts import ImageArtifact
|
|
16
|
+
from argus.measurement import measurement_scope
|
|
15
17
|
|
|
16
18
|
from .catalog import (
|
|
17
19
|
_AGENT_TOOL_DESCRIPTIONS,
|
|
@@ -22,7 +24,9 @@ from .catalog import (
|
|
|
22
24
|
_SCREEN_TOOL_NAMES,
|
|
23
25
|
_TOOL_TITLES,
|
|
24
26
|
)
|
|
27
|
+
from .overhead import response_bytes
|
|
25
28
|
from .results import AgentResult, to_mcp_result
|
|
29
|
+
from .snapshots import FIELDS, ObservationSnapshots
|
|
26
30
|
|
|
27
31
|
INSTRUCTIONS = (
|
|
28
32
|
"Start with start_session; follow its protocol. Exercise real journeys and "
|
|
@@ -60,16 +64,29 @@ def tool_names(profile: str) -> set[str]:
|
|
|
60
64
|
raise ValueError("tool profile must be agent, core, screen, or full")
|
|
61
65
|
|
|
62
66
|
|
|
63
|
-
def bind_tool(runtime: SessionManager, operation):
|
|
67
|
+
def bind_tool(runtime: SessionManager, operation, snapshots: ObservationSnapshots | None = None):
|
|
64
68
|
@wraps(operation)
|
|
65
69
|
async def call(*args, ctx: Context | None = None, **kwargs):
|
|
66
70
|
async def notify(sequence: int, message: str) -> None:
|
|
67
71
|
if ctx is not None:
|
|
68
72
|
await ctx.report_progress(sequence, message=message)
|
|
69
73
|
|
|
70
|
-
with
|
|
71
|
-
|
|
72
|
-
|
|
74
|
+
with measurement_scope() as measurements:
|
|
75
|
+
with progress_scope(notify if ctx is not None else None) as progress:
|
|
76
|
+
try:
|
|
77
|
+
result = await runtime.execute(operation.__name__, operation, *args, **kwargs)
|
|
78
|
+
except BaseException:
|
|
79
|
+
if snapshots is not None:
|
|
80
|
+
snapshots.clear()
|
|
81
|
+
raise
|
|
82
|
+
encoded = to_mcp_result(
|
|
83
|
+
operation.__name__, result, progress, runtime.settings.response_mode, snapshots
|
|
84
|
+
)
|
|
85
|
+
runtime.response_totals.update(response_bytes(encoded))
|
|
86
|
+
runtime.response_totals["responses"] += 1
|
|
87
|
+
runtime.component_seconds.update(measurements.seconds)
|
|
88
|
+
runtime.component_calls.update(measurements.calls)
|
|
89
|
+
return encoded
|
|
73
90
|
|
|
74
91
|
signature = inspect.signature(operation, eval_str=True)
|
|
75
92
|
parameters = [
|
|
@@ -88,10 +105,21 @@ def bind_tool(runtime: SessionManager, operation):
|
|
|
88
105
|
|
|
89
106
|
def create_server(runtime: SessionManager, profile: str = "core") -> FastMCP:
|
|
90
107
|
server = FastMCP("argus", instructions=INSTRUCTIONS)
|
|
108
|
+
# FastMCP 1.x does not expose the low-level server's product version argument.
|
|
109
|
+
server._mcp_server.version = __version__
|
|
110
|
+
snapshots = ObservationSnapshots()
|
|
111
|
+
if runtime.settings.response_mode == "delta":
|
|
112
|
+
server._mcp_server.instructions += (
|
|
113
|
+
" Delta mode: retain observation snapshots by ID. In delta snapshots, inherit only "
|
|
114
|
+
+ ", ".join(sorted(FIELDS))
|
|
115
|
+
+ "; replace returned fields and delete snapshot.removed fields. base_id must match. "
|
|
116
|
+
"Other fields are current-call-only. On a missing base call observe for full state. "
|
|
117
|
+
"Never infer unchanged from an unavailable base."
|
|
118
|
+
)
|
|
91
119
|
for name in sorted(tool_names(profile)):
|
|
92
120
|
read_only = name in _READ_ONLY_TOOL_NAMES
|
|
93
121
|
server.add_tool(
|
|
94
|
-
bind_tool(runtime, OPERATIONS[name]),
|
|
122
|
+
bind_tool(runtime, OPERATIONS[name], snapshots),
|
|
95
123
|
name=name,
|
|
96
124
|
description=_AGENT_TOOL_DESCRIPTIONS[name] if profile == "agent" else None,
|
|
97
125
|
title=_TOOL_TITLES[name],
|