argus-testing 0.12.2__tar.gz → 0.13.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {argus_testing-0.12.2 → argus_testing-0.13.0}/.gitignore +1 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/AGENTS.md +13 -4
- {argus_testing-0.12.2 → argus_testing-0.13.0}/CHANGELOG.md +179 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/CONTRIBUTING.md +22 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/PKG-INFO +9 -6
- {argus_testing-0.12.2 → argus_testing-0.13.0}/README.md +8 -5
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/_compat.py +3 -1
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/access.py +4 -1
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/adapters/journal.py +2 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/adapters/monitoring.py +4 -5
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/adapters/reporting/common.py +14 -2
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/adapters/reporting/html.py +10 -2
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/adapters/reporting/junit.py +16 -6
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/adapters/reporting/sarif.py +10 -2
- argus_testing-0.13.0/argus/adapters/scripts/collect_text_js.js +136 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/adapters/scripts/extract_elements_js.js +53 -5
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/adapters/scripts/extract_page_content_js.js +21 -12
- argus_testing-0.13.0/argus/adapters/scripts/ref_engine_js.js +16 -0
- argus_testing-0.13.0/argus/adapters/scripts/row_text_js.js +26 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/application/actions.py +43 -28
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/application/coverage.py +16 -7
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/application/evidence.py +7 -3
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/application/expectations.py +7 -5
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/application/exports.py +10 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/application/findings.py +4 -4
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/application/inspection.py +4 -1
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/application/journeys.py +40 -5
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/application/lifecycle.py +20 -12
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/application/manager.py +9 -1
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/application/navigation.py +4 -1
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/application/network_storage.py +15 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/application/observation.py +13 -6
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/application/probes.py +88 -27
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/application/regression.py +54 -6
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/application/screen.py +110 -59
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/application/screenshots.py +8 -7
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/application/testgen.py +178 -81
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/application/text.py +54 -52
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/application/verification.py +89 -57
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/browser.py +218 -64
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/cleanup.py +10 -2
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/cli.py +29 -1
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/differ.py +5 -3
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/domain/models.py +22 -0
- argus_testing-0.13.0/argus/domain/verdict.py +345 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/domain/verification.py +2 -12
- argus_testing-0.13.0/argus/domain/wordchars.py +6 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/mcp_server.py +9 -2
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/ports.py +2 -1
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/resolver.py +64 -22
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/screen/backend.py +250 -106
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/screen/safety.py +29 -17
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/security.py +27 -15
- {argus_testing-0.12.2 → argus_testing-0.13.0}/docs/agent-protocol.md +78 -16
- {argus_testing-0.12.2 → argus_testing-0.13.0}/docs/architecture.md +7 -4
- {argus_testing-0.12.2 → argus_testing-0.13.0}/docs/decisions.md +19 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/docs/evaluation.md +1 -1
- {argus_testing-0.12.2 → argus_testing-0.13.0}/docs/usage.md +24 -6
- {argus_testing-0.12.2 → argus_testing-0.13.0}/examples/response-overhead.json +3 -3
- {argus_testing-0.12.2 → argus_testing-0.13.0}/llms-install.md +7 -2
- {argus_testing-0.12.2 → argus_testing-0.13.0}/pyproject.toml +1 -1
- {argus_testing-0.12.2 → argus_testing-0.13.0}/server.json +2 -2
- argus_testing-0.13.0/tests/test_action_time_resolution.py +159 -0
- argus_testing-0.13.0/tests/test_audit_hardening.py +276 -0
- argus_testing-0.13.0/tests/test_crawl_site.py +85 -0
- argus_testing-0.13.0/tests/test_element_identity.py +154 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_evidence_provenance.py +21 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_export_guards.py +34 -4
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_export_privacy.py +20 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_export_test.py +180 -1
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_final_hardening.py +4 -0
- argus_testing-0.13.0/tests/test_frame_extraction.py +116 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_mcp_contract.py +24 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_monitoring.py +10 -1
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_release_metadata.py +3 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_reproduction_receipt.py +4 -2
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_resolver.py +107 -0
- argus_testing-0.13.0/tests/test_row_textarea.py +127 -0
- argus_testing-0.13.0/tests/test_screen_errors.py +154 -0
- argus_testing-0.13.0/tests/test_screen_input_guard.py +194 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_security_boundaries.py +21 -0
- argus_testing-0.13.0/tests/test_textarea_label.py +85 -0
- argus_testing-0.13.0/tests/test_verdict.py +177 -0
- argus_testing-0.13.0/tests/test_verdict_integrity.py +365 -0
- argus_testing-0.13.0/tests/test_verdict_parity.py +405 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_verification_contract.py +2 -2
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_verification_efficiency.py +33 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/LICENSE +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/PRIVACY.md +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/SECURITY.md +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/__init__.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/adapters/__init__.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/adapters/assets/__init__.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/adapters/assets/dashboard.css +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/adapters/assets/dashboard.html +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/adapters/assets/dashboard.js +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/adapters/logging.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/adapters/reporting/__init__.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/adapters/reporting/json.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/adapters/reporting/privacy.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/adapters/reporting/reporter.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/adapters/scripts/__init__.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/adapters/scripts/check_layout_js.js +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/adapters/scripts/extract_performance_js.js +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/adapters/scripts/find_visible_element_js.js +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/adapters/scripts/inspect_element_js.js +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/application/__init__.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/application/capsules.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/application/capture.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/application/context.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/application/events.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/application/finish.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/application/guidance.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/application/operations.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/application/payloads.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/application/progress.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/application/session.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/application/telemetry.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/application/verifiers.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/bootstrap.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/detector.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/doctor.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/domain/__init__.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/domain/contracts.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/domain/coverage.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/domain/detector.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/domain/findings.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/domain/replay.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/entrypoints/__init__.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/entrypoints/mcp/__init__.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/entrypoints/mcp/catalog.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/entrypoints/mcp/compat.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/entrypoints/mcp/overhead.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/entrypoints/mcp/results.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/entrypoints/mcp/server.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/entrypoints/mcp/snapshots.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/install_check.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/measurement.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/models.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/reporter.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/runtime.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/screen/__init__.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/screen/permissions.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/screen/validate.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/settings.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus/storage.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus_lab/__init__.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus_lab/bench/__init__.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus_lab/bench/__main__.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus_lab/bench/runner.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus_lab/bench/scenarios_buggytasks.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus_lab/bench/scenarios_darkshop.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus_lab/evaluation/__init__.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus_lab/evaluation/__main__.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus_lab/evaluation/adjudication.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus_lab/evaluation/agent_fixture.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus_lab/evaluation/browser_baseline.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus_lab/evaluation/codex_runner.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus_lab/evaluation/fixture.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus_lab/evaluation/gateway.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus_lab/evaluation/models.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus_lab/evaluation/overhead.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus_lab/evaluation/scoring.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/argus_lab/evaluation/smoke.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/docs/evaluation-0.8.md +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/examples/agent-evaluation-optimized-evidence.json +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/examples/agent-evaluation-optimized.json +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/examples/agent-evaluation-pilot-evidence.json +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/examples/agent-evaluation-pilot.json +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/examples/mcp-config.json +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/glama.json +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/human-eye-fixture/app.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/scripts/check_distributions.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/test-site/app.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/__init__.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/conftest.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_action_targeting.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_adjudication.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_agent_results.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_agent_workflow.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_architecture_runtime.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_bench_report.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_browser_shadow.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_codex_evaluation.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_cross_stack.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_deprecated_modules.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_detector_console.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_detector_network.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_distribution_contents.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_doctor.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_evaluation.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_expectation_oracle.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_final_coverage.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_form_checkboxes.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_journal_regression.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_mcp_compat.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_real_app_interactions.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_regression_cli.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_replay_minimize.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_replay_receipt.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_replay_trace.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_reporter_badge.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_response_efficiency.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_runtime_limits.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_screen_permissions.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_screen_safety.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_session_lifecycle.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_state_capsules.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_stdio_protocol.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_storage_cleanup.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_substrate.py +0 -0
- {argus_testing-0.12.2 → argus_testing-0.13.0}/tests/test_visual_inspection.py +0 -0
|
@@ -84,7 +84,7 @@ result includes captured stages. See [the protocol](docs/agent-protocol.md).
|
|
|
84
84
|
| `test_action(target, expect=...)` | Click + before/after diff in one call. Pass `expect` to PREDICT the outcome ({"count":{"label":"tasks","delta":1}}, {"gains":"Buy milk"}, {"removes":...}, {"text_present":...}, {"toast":...}, {"url_changed":true}) and Argus reports MATCH / SURPRISE — a surprise is a bug lead. Also shows CROSS-STACK: which requests the click fired (methods/statuses) and a CHECK nudge when a message appeared without a matching write. |
|
|
85
85
|
| `verify_persistence(expect, target_text, after_url)` | Forces a fresh GET; returns target presence and the page observation. Check persistence once, repeat only to resolve uncertainty. A matching symptom is not proof of a semantic defect. |
|
|
86
86
|
| `capsule_save(name, liveness_marker)` / `capsule_restore(name)` | Snapshot cookies and page storage after establishing state through the UI. Restore replaces the captured client state and checks a saved liveness marker when provided; without one, validity remains unverified. It does not reset backend data or guarantee identical application state across journeys. |
|
|
87
|
-
| `regression_check()` | Re-test
|
|
87
|
+
| `regression_check()` | Re-test journaled findings against the current build: `STILL-PRESENT`, `FIXED` (the recorded symptom no longer reproduces), or `INCONCLUSIVE`. Mixed receipts remain inconclusive; a `FIXED` result is not proof of application correctness. |
|
|
88
88
|
| `export_test(finding_id, run=False, save_private_values=False)` | Export a standalone Playwright test from a finding verified by a fresh-load text check. Confirm it fails on the symptom before using it in CI. Unsupported steps are marked TODO; 0.12.1 blocks incomplete exports before execution. See the [export protocol](docs/agent-protocol.md#exporting-a-regression-test) for environment variables and package-version limits. `run=true` repeats the journey's writes, so use it only with the user's authority. Only set `save_private_values=true` when the user explicitly requests retaining local replay values; its returned JSON file can contain credentials and must never be published. |
|
|
89
89
|
| `inspect_element(description)` | Computed styles + ARIA + outerHTML + truncation flag for interactive or visible non-interactive content. |
|
|
90
90
|
| `check_layout()` | Bounded viewport signals for horizontal overflow, clipped text, small targets, and fixed/sticky overlays. |
|
|
@@ -98,7 +98,7 @@ result includes captured stages. See [the protocol](docs/agent-protocol.md).
|
|
|
98
98
|
| `crawl_site()` | Page discovery: crawls internal links, auto-capturing only console/network events (tagged). Walk the surfaced pages and record_bug what you confirm. |
|
|
99
99
|
| `end_session(coverage_updates=...)` | Optionally submit explicit final exercised/blocked goals with evidence, then close and export reports. The entire batch validates before shutdown; invalid updates keep the session open. |
|
|
100
100
|
|
|
101
|
-
### Screen mode (macOS)
|
|
101
|
+
### Screen mode (macOS, experimental)
|
|
102
102
|
|
|
103
103
|
| Tool | Purpose |
|
|
104
104
|
|------|---------|
|
|
@@ -221,8 +221,11 @@ Don't record speculation, polish nits, or static a11y / SEO that
|
|
|
221
221
|
- Session cap: 30 min default
|
|
222
222
|
(`ARGUS_SCREEN_SESSION_MAX_SECONDS`).
|
|
223
223
|
- Panic button: `touch ~/.argus/abort` blocks every subsequent screen
|
|
224
|
-
action
|
|
225
|
-
|
|
224
|
+
action, including `screen_launch` and `screen_quit`, and no new screen
|
|
225
|
+
session starts while the file exists. Only the user removes it; never
|
|
226
|
+
delete it or work around it (`ARGUS_SCREEN_ABORT_FILE` to relocate).
|
|
227
|
+
- Keyboard and coordinate input is sent only while the bound app is
|
|
228
|
+
frontmost. Argus brings it forward first and refuses if that fails.
|
|
226
229
|
- Every `screen_click_what` / `screen_type_into` writes a
|
|
227
230
|
before-and-after screenshot pair to the action trail.
|
|
228
231
|
|
|
@@ -251,6 +254,12 @@ application data.
|
|
|
251
254
|
journal), 1 for still-present findings, 2 for operational/storage failures and
|
|
252
255
|
3 for inconclusive checks. An empty journal is not proof of app correctness.
|
|
253
256
|
|
|
257
|
+
Since 0.12.2, automatic regression refuses storage-reset checks and legacy entries
|
|
258
|
+
without complete preconditions. Preserve journals from 0.12.1 and earlier and
|
|
259
|
+
re-record findings in a new output directory. Regenerate older exported tests;
|
|
260
|
+
see [migration guidance](docs/usage.md#regression-gates-in-ci). Never drop a
|
|
261
|
+
storage-reset precondition to make a finding exportable.
|
|
262
|
+
|
|
254
263
|
## What changed in 0.5 (if you knew 0.4)
|
|
255
264
|
|
|
256
265
|
- `get_page_state` → `observe` (no integer indices).
|
|
@@ -1,5 +1,184 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## Plugin updates - 2026-11-10
|
|
4
|
+
|
|
5
|
+
- Claude plugin 0.12.4 and Codex plugin 0.12.3 pin the published Argus 0.12.3
|
|
6
|
+
runtime and refresh their dependency locks and setup commands.
|
|
7
|
+
|
|
8
|
+
## 0.13.0 - 2026-11-12
|
|
9
|
+
|
|
10
|
+
One set of verdict rules. Live verification, `verify_persistence` and exported
|
|
11
|
+
tests decided "is the symptom present?" with three separate implementations
|
|
12
|
+
that had drifted apart; they now run the same code. No new tools.
|
|
13
|
+
|
|
14
|
+
Upgrading: regenerate exported tests to get the shared rules (earlier files
|
|
15
|
+
keep working with their old rules). "Present" now means text a user can see or
|
|
16
|
+
the value of a visible field, so text reachable only through `aria-label`,
|
|
17
|
+
unselected `<option>` text and closed `<details>` content no longer count.
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
- Description-driven web tools (`click_what`, `type_into`, `test_action`,
|
|
21
|
+
`inspect_element` and the rest) resolve the description against the page as
|
|
22
|
+
it is when the action runs, not against the last observation: a row that
|
|
23
|
+
re-rendered in place is found by its current text, and a description that has
|
|
24
|
+
become ambiguous or no longer matches returns candidates or a no-match error
|
|
25
|
+
instead of acting.
|
|
26
|
+
- Screen-mode tools report failures and refusals as errors with a code
|
|
27
|
+
(`permission_denied` for safety refusals and missing macOS grants, `timeout`,
|
|
28
|
+
`invalid_argument`, `backend_error`) instead of completed calls.
|
|
29
|
+
- A `<textarea>` is named by its aria-label, `<label>`, placeholder or name, and
|
|
30
|
+
its initial content is reported only as its value, so a secret-named field's
|
|
31
|
+
content is hidden as `(value hidden)` instead of shown as its label.
|
|
32
|
+
- Row text separates adjacent elements that the markup writes without
|
|
33
|
+
whitespace, as frameworks usually do. `<span>Buy groceries</span><button>Delete</button>`
|
|
34
|
+
read as "Buy groceriesDelete", so "Delete near Buy groceries" matched nothing.
|
|
35
|
+
- A row's text (the `near:` context, repeating item lists and exported tests'
|
|
36
|
+
row matching) leaves out `<textarea>` content, so a secret in a textarea no
|
|
37
|
+
longer shows through the row of every control beside it. Dialog previews,
|
|
38
|
+
toasts and the focused element follow the same rule, and the focused
|
|
39
|
+
element's value is no longer collected.
|
|
40
|
+
- Fresh-load verification and `verify_persistence` read the same visible text:
|
|
41
|
+
the page and every visible frame (cross-origin frames and open shadow roots
|
|
42
|
+
included) plus the values of visible fields. Text that is only inside an
|
|
43
|
+
iframe is now found, and hidden text inside a visible list row or element no
|
|
44
|
+
longer counts as present.
|
|
45
|
+
- Login-wall phrases are searched in the main frame's visible text, so a sign-in
|
|
46
|
+
banner rendered in a shadow root makes an absent symptom inconclusive.
|
|
47
|
+
- `verify_persistence` suspends the session's network mocks for its fresh load
|
|
48
|
+
and restores them afterwards, so it can no longer confirm a response the agent
|
|
49
|
+
injected, and reports an absent target on a page whose network never settles
|
|
50
|
+
as inconclusive.
|
|
51
|
+
- Exported regression tests embed Argus's verdict rules and visible-text
|
|
52
|
+
collector verbatim (the docstring names the rules version), so they decide a
|
|
53
|
+
symptom exactly as fresh-load verification and `verify_persistence` do:
|
|
54
|
+
unspaced scripts such as CJK match, hidden field values do not count, open
|
|
55
|
+
shadow roots are read, and a redirect elsewhere or a non-English login form
|
|
56
|
+
makes an absent symptom inconclusive.
|
|
57
|
+
- `export_test(run=True)` reports a new outcome, `inconclusive`, with a reason
|
|
58
|
+
code, for an error page, login wall, redirect, unsettled or unreadable page;
|
|
59
|
+
these were `errored` before. The file's last stderr line starts with
|
|
60
|
+
`ARGUS_INCONCLUSIVE`; the test fails under pytest and the file exits 3 when
|
|
61
|
+
run as a script. Earlier exports keep their old rules until regenerated.
|
|
62
|
+
- A child frame that cannot be read now gives its own inconclusive reason
|
|
63
|
+
instead of claiming the page text exceeded the inspection limit.
|
|
64
|
+
- Capsule liveness markers, `test_action`'s `text_present`/`text_absent` and the
|
|
65
|
+
action diff read the same visible frame text.
|
|
66
|
+
|
|
67
|
+
## 0.12.3 - 2026-11-10
|
|
68
|
+
|
|
69
|
+
Fixes from a full audit of 0.12.2. Journals and exported tests stay compatible.
|
|
70
|
+
Some checks that returned `FIXED` or `verified` on evidence that could not show
|
|
71
|
+
the symptom now return inconclusive, and failed actions are reported as errors.
|
|
72
|
+
Screen mode is experimental.
|
|
73
|
+
|
|
74
|
+
- Check symptom text against the whole page instead of its first 5,000
|
|
75
|
+
characters, so text further down is no longer reported as gone. Match targets
|
|
76
|
+
inside Chinese, Japanese, Korean and Thai text. Keep lists that share a class
|
|
77
|
+
or tag apart.
|
|
78
|
+
- Load a new document for fresh-load checks of hash-route URLs; a same-document
|
|
79
|
+
navigation previously read in-memory state.
|
|
80
|
+
- Treat an HTTP-status check as inconclusive when the load answers with a
|
|
81
|
+
different error, is redirected or lands on a login page. Only a clean answer
|
|
82
|
+
from the requested page counts as no longer reproduced.
|
|
83
|
+
- Treat an absent symptom as inconclusive when the load ended on another page
|
|
84
|
+
or shows a login form in any common UI language.
|
|
85
|
+
- `argus-regression` exits 2 for a URL without a scheme or with credentials and
|
|
86
|
+
for journal entries of the wrong type, and names other origins that have
|
|
87
|
+
journals when the given one has none.
|
|
88
|
+
- Report intermittent receipts as inconclusive in JUnit (a failure, labelled
|
|
89
|
+
INTERMITTENT), SARIF and the HTML badge. Map SARIF `critical` to `error`.
|
|
90
|
+
Strip characters XML cannot carry. Show inconclusive checks as INCONCLUSIVE in
|
|
91
|
+
HTML. Redact regression artifacts.
|
|
92
|
+
- `argus-cleanup` ignores report files whose name is not a run ID; one such
|
|
93
|
+
name could select the whole output directory. Automatic screenshot names can
|
|
94
|
+
no longer leave the run directory.
|
|
95
|
+
- Make credential redaction linear in input size and cap captured request
|
|
96
|
+
bodies. Stop redacting ordinary keys such as `author` and `sidebar`; redact
|
|
97
|
+
`pwd`, quoted values, full Bearer and Basic values, signatures and private keys.
|
|
98
|
+
- Observe forms that contain a control named `id`. Attach page listeners once.
|
|
99
|
+
Accept unsaved-changes prompts so navigation proceeds. Recover after a failed
|
|
100
|
+
navigation. Judge internal links by origin.
|
|
101
|
+
- Act on the exact element that was listed. Each observed element keeps a
|
|
102
|
+
reference to its DOM node, so a click or fill no longer lands on an earlier
|
|
103
|
+
control that shares a label prefix, on a hidden hover-only control in another
|
|
104
|
+
row, or on a same-named field in another form. An element that has been
|
|
105
|
+
removed fails the action instead of redirecting it.
|
|
106
|
+
- `test_form` with automatic submit only considers controls in the form that
|
|
107
|
+
was filled.
|
|
108
|
+
- Screen mode: send keyboard and coordinate input only while the bound app is
|
|
109
|
+
frontmost; a session started without a target stays with its first app.
|
|
110
|
+
The abort file now also stops `screen_launch` and `screen_quit`, and a new
|
|
111
|
+
session no longer deletes it: only the user clears it. `screen_quit` needs an
|
|
112
|
+
exact or unique app match. Hold, drag and click counts are bounded, key
|
|
113
|
+
combinations apply modifiers per key and accept hyphenated key names, and
|
|
114
|
+
failed typing no longer echoes the text. `screen_wait_for_stable` works.
|
|
115
|
+
`ARGUS_SCREEN_ABORT_FILE` expands `~`, and its directory is created at start.
|
|
116
|
+
- `test_action` and `test_form` return the console and HTTP failures an action
|
|
117
|
+
raised, and the cross-stack check, as structured data (`browser_events`,
|
|
118
|
+
`cross_stack`); compact and delta responses previously dropped them.
|
|
119
|
+
- Failed actions (hover, resize, upload, drag, coordinate input, missing
|
|
120
|
+
screenshots, a screen session that cannot start) are reported as errors
|
|
121
|
+
instead of `status: completed`.
|
|
122
|
+
- `argus-regression` gives each finding its own deadline; a long journal or one
|
|
123
|
+
unreachable page no longer fails the whole run.
|
|
124
|
+
- Coverage goals match on whole words.
|
|
125
|
+
- Network mocks are registered on the browser context with one handler per
|
|
126
|
+
pattern. Replacing a mock no longer leaves the first response active during
|
|
127
|
+
a fresh-load verification, and mocks follow tab switches and device emulation.
|
|
128
|
+
- CI also runs on Python 3.13 and 3.14.
|
|
129
|
+
- Screen mode reads the frontmost app from the accessibility and window
|
|
130
|
+
servers. `NSWorkspace` is only refreshed by a running main run loop, so a
|
|
131
|
+
long-lived server kept seeing the app that was frontmost at startup and
|
|
132
|
+
refused all input. `screen_quit` and `screen_launch` report a name that
|
|
133
|
+
matches several running apps instead of "not running".
|
|
134
|
+
- A start rejected after launch closes the browser; `end_session` always leaves
|
|
135
|
+
the failed state. `test_form` no longer echoes a typed value when a fill fails.
|
|
136
|
+
- Resolve "Last name" and "First name" to the fields with those labels instead
|
|
137
|
+
of reading the word as a position. Only the last kind word sets the kind, so
|
|
138
|
+
"Select all button" and "Menu button" pick the button. A bare label shared by
|
|
139
|
+
several rows stays ambiguous. Chinese, Japanese, Korean and Thai descriptions
|
|
140
|
+
match inside longer labels.
|
|
141
|
+
- List elements in `srcdoc` and blank iframes once, and reach controls in
|
|
142
|
+
cross-origin frames nested inside other cross-origin frames.
|
|
143
|
+
- An exported step located by a structural CSS path (`:nth-of-type`,
|
|
144
|
+
`:nth-child`) is positional: the export is incomplete and does not run.
|
|
145
|
+
- `drag_what`, `right_click`, `storage_set`, `storage_remove`, `storage_clear`,
|
|
146
|
+
`cookies_set`, `cookies_clear`, `tabs_switch` and `eval_js` appear in the
|
|
147
|
+
action trace without their values, so an export that contains them is
|
|
148
|
+
incomplete and a replay through them reports that it cannot repeat them.
|
|
149
|
+
- `crawl_site` stays on the origin it started from, skips a link that redirects
|
|
150
|
+
elsewhere, returns to the start page, and counts only the event-bugs it captured.
|
|
151
|
+
- `network_mock`, `network_unmock`, `network_clear_mocks`, `tabs_close` and
|
|
152
|
+
`set_dialog_handler` appear in the action trace, the mock without its body and
|
|
153
|
+
the dialog handler without its text, so an export after a mock is incomplete.
|
|
154
|
+
- A recheck that verifies a finding recorded without a goal updates the coverage
|
|
155
|
+
evidence of every goal whose window holds it, not only of tagged goals.
|
|
156
|
+
- Element descriptions hide the value of a text field whose name, id, label or
|
|
157
|
+
placeholder names a secret, such as `api_key`.
|
|
158
|
+
- `wait_for_request` redacts credentials in the matched URL.
|
|
159
|
+
- The monitor rejects same-site requests, such as an image on the app under test
|
|
160
|
+
pointing at the event stream; only same-origin, direct and non-browser
|
|
161
|
+
requests are accepted.
|
|
162
|
+
- An empty `ARGUS_TOOL_PROFILE` means `web`, and an invalid one is reported as a
|
|
163
|
+
usage error naming the valid profiles instead of a traceback.
|
|
164
|
+
|
|
165
|
+
## Documentation and Glama deployment - 2026-11-09
|
|
166
|
+
|
|
167
|
+
- Synchronize the agent installation guide and protocol with Argus 0.12.2,
|
|
168
|
+
including journal migration, regression verdicts and export preconditions.
|
|
169
|
+
- Publish Glama 0.12.2 from release commit
|
|
170
|
+
`e2ac33da3b3eda998c3ba98084c33ed291381b66` after adding `libatomic1` to its
|
|
171
|
+
build steps. The verified build reports MCP server version 0.12.2.
|
|
172
|
+
- Document the system dependency and the wait between build success and image
|
|
173
|
+
availability for release creation. The runtime package version is unchanged.
|
|
174
|
+
|
|
175
|
+
## Plugin updates - 2026-11-08
|
|
176
|
+
|
|
177
|
+
- Claude plugin 0.12.3 and Codex plugin 0.12.2 pin the published Argus 0.12.2
|
|
178
|
+
runtime and refresh their dependency locks and setup commands.
|
|
179
|
+
- Regenerate older exported tests to use the regression reliability fixes;
|
|
180
|
+
keep legacy journals and re-record findings in a new report directory.
|
|
181
|
+
|
|
3
182
|
## 0.12.2 - 2026-11-08
|
|
4
183
|
|
|
5
184
|
- Treat intermittent regression receipts as inconclusive instead of fixed.
|
|
@@ -90,6 +90,11 @@ For release, synchronize `pyproject.toml` and both version fields in
|
|
|
90
90
|
`server.json`, update `CHANGELOG.md`, build with `python -m build`, check with
|
|
91
91
|
`twine check`, and smoke-test the wheel outside the source directory. JS and
|
|
92
92
|
dashboard assets are package resources and must be present in the wheel.
|
|
93
|
+
Update the current-release section in `README.md` and the version and install pin
|
|
94
|
+
in `llms-install.md`. When behavior changes, align `docs/usage.md`,
|
|
95
|
+
`docs/agent-protocol.md`, architecture and privacy guidance with the implemented
|
|
96
|
+
contract. Keep historical version notes and evaluation results tied to the
|
|
97
|
+
versions they actually describe.
|
|
93
98
|
Source distributions use an explicit inclusion list. Run
|
|
94
99
|
`python scripts/check_distributions.py dist/*` after building; newly packaged
|
|
95
100
|
files must satisfy the public-content policy before publication.
|
|
@@ -106,6 +111,23 @@ publication succeed, manually publish that verified Glama build under the same
|
|
|
106
111
|
version. A version label does not prove which source was built. Do not reuse an
|
|
107
112
|
occupied version label for a different build.
|
|
108
113
|
|
|
114
|
+
For Glama's `debian:bookworm-slim` build, install `libatomic1` before invoking
|
|
115
|
+
Playwright. Its bundled Node driver needs `libatomic.so.1` to start, including
|
|
116
|
+
when running `playwright install --with-deps`. Use these build steps:
|
|
117
|
+
|
|
118
|
+
```json
|
|
119
|
+
[
|
|
120
|
+
"apt-get update && apt-get install -y --no-install-recommends libatomic1",
|
|
121
|
+
"uv sync",
|
|
122
|
+
"uv run playwright install --with-deps chromium"
|
|
123
|
+
]
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
A successful build can precede the image becoming available in Glama's registry.
|
|
127
|
+
If release creation reports that the image is not available yet, wait and retry
|
|
128
|
+
the same verified build. Confirm the published release links to that build;
|
|
129
|
+
do not treat a successful test alone as a published release.
|
|
130
|
+
|
|
109
131
|
After publication, verify the PyPI wheel and source archive, official MCP
|
|
110
132
|
Registry package version, and directory release associations. Run
|
|
111
133
|
`python -m argus.install_check` against the published package outside the checkout.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: argus-testing
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.13.0
|
|
4
4
|
Summary: Testing MCP for coding agents: exercise real user journeys, find bugs, and verify them with evidence and persistence checks
|
|
5
5
|
Project-URL: Homepage, https://yichenwu.dev/projects/argus
|
|
6
6
|
Project-URL: Documentation, https://github.com/chriswu727/argus#readme
|
|
@@ -129,11 +129,14 @@ replaying a journey can repeat its writes.
|
|
|
129
129
|
[Cursor, other MCP clients and troubleshooting](docs/usage.md#quick-start) ·
|
|
130
130
|
[Claude plugin](plugins/claude) · [Codex plugin](plugins/codex)
|
|
131
131
|
|
|
132
|
-
**0.
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
132
|
+
**0.13.0**: live verification, `verify_persistence` and exported tests now
|
|
133
|
+
decide with the same rules, checked against each other on every change. They
|
|
134
|
+
read what a user can see, in every frame and open shadow root, and an exported
|
|
135
|
+
test reports an inconclusive check instead of a false failure. Regenerate
|
|
136
|
+
exported tests to get the shared rules; see the
|
|
137
|
+
[journal migration steps](docs/usage.md#regression-gates-in-ci), the
|
|
138
|
+
[releases](https://github.com/chriswu727/argus/releases) and the
|
|
139
|
+
[changelog](CHANGELOG.md#0130---2026-11-12).
|
|
137
140
|
|
|
138
141
|
## Engineering
|
|
139
142
|
|
|
@@ -85,11 +85,14 @@ replaying a journey can repeat its writes.
|
|
|
85
85
|
[Cursor, other MCP clients and troubleshooting](docs/usage.md#quick-start) ·
|
|
86
86
|
[Claude plugin](plugins/claude) · [Codex plugin](plugins/codex)
|
|
87
87
|
|
|
88
|
-
**0.
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
88
|
+
**0.13.0**: live verification, `verify_persistence` and exported tests now
|
|
89
|
+
decide with the same rules, checked against each other on every change. They
|
|
90
|
+
read what a user can see, in every frame and open shadow root, and an exported
|
|
91
|
+
test reports an inconclusive check instead of a false failure. Regenerate
|
|
92
|
+
exported tests to get the shared rules; see the
|
|
93
|
+
[journal migration steps](docs/usage.md#regression-gates-in-ci), the
|
|
94
|
+
[releases](https://github.com/chriswu727/argus/releases) and the
|
|
95
|
+
[changelog](CHANGELOG.md#0130---2026-11-12).
|
|
93
96
|
|
|
94
97
|
## Engineering
|
|
95
98
|
|
|
@@ -92,7 +92,6 @@ from argus.application.text import _short as _short
|
|
|
92
92
|
from argus.application.text import _text_in_state as _text_in_state
|
|
93
93
|
from argus.application.text import _token_present as _token_present
|
|
94
94
|
from argus.application.text import _visible_text_in_state as _visible_text_in_state
|
|
95
|
-
from argus.application.verification import _LOGOUT_PHRASES as _LOGOUT_PHRASES
|
|
96
95
|
from argus.application.verification import _looks_logged_out as _looks_logged_out
|
|
97
96
|
from argus.application.verification import _minimize_replay as _minimize_replay
|
|
98
97
|
from argus.application.verification import _receipt_verdict as _receipt_verdict
|
|
@@ -100,3 +99,6 @@ from argus.application.verification import _run_replay_receipt as _run_replay_re
|
|
|
100
99
|
from argus.application.verification import (
|
|
101
100
|
_run_reproduction_check as _run_reproduction_check,
|
|
102
101
|
)
|
|
102
|
+
from argus.domain.verdict import LOGIN_WALL_PHRASES
|
|
103
|
+
|
|
104
|
+
_LOGOUT_PHRASES = LOGIN_WALL_PHRASES
|
|
@@ -84,7 +84,10 @@ class AccessPolicy:
|
|
|
84
84
|
return resolved
|
|
85
85
|
|
|
86
86
|
def check_url(self, url: str) -> None:
|
|
87
|
-
|
|
87
|
+
# Chromium's own error page (failed navigation) holds no site content;
|
|
88
|
+
# rejecting it would refuse every later call, including the navigate
|
|
89
|
+
# that leaves it.
|
|
90
|
+
if url == "about:blank" or url.startswith("chrome-error://"):
|
|
88
91
|
return
|
|
89
92
|
try:
|
|
90
93
|
parsed = urlsplit(url)
|
|
@@ -17,6 +17,8 @@ class FileJournal:
|
|
|
17
17
|
if any(
|
|
18
18
|
not isinstance(entry.get("fingerprint"), str)
|
|
19
19
|
or not isinstance(entry.get("verify"), dict)
|
|
20
|
+
or not isinstance(entry.get("title", ""), str)
|
|
21
|
+
or not isinstance(entry.get("severity", ""), str)
|
|
20
22
|
for entry in entries
|
|
21
23
|
):
|
|
22
24
|
raise StorageError(f"invalid finding entry in {path}")
|
|
@@ -61,11 +61,10 @@ class Monitor:
|
|
|
61
61
|
)
|
|
62
62
|
except ValueError:
|
|
63
63
|
return False
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
)
|
|
64
|
+
# Another port on localhost is same-site, and a page served there (usually
|
|
65
|
+
# the app under test) must not be able to hold the event stream slots.
|
|
66
|
+
fetch_site = request.headers.get("sec-fetch-site")
|
|
67
|
+
return host in allowed and valid_origin and fetch_site in (None, "same-origin", "none")
|
|
69
68
|
|
|
70
69
|
async def health(self, request: Request) -> Response:
|
|
71
70
|
return JSONResponse({"status": "ok", "schema_version": 1})
|
|
@@ -2,6 +2,7 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import base64
|
|
4
4
|
import hashlib
|
|
5
|
+
import re
|
|
5
6
|
from pathlib import Path
|
|
6
7
|
from typing import List, Optional
|
|
7
8
|
from urllib.parse import quote
|
|
@@ -14,6 +15,17 @@ from argus.domain.models import (
|
|
|
14
15
|
Severity,
|
|
15
16
|
)
|
|
16
17
|
|
|
18
|
+
_XML_ILLEGAL = re.compile("[^\t\n\r\u0020-\ud7ff\ue000-\ufffd\U00010000-\U0010ffff]")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def xml_safe(document: str) -> str:
|
|
22
|
+
"""Drop characters XML 1.0 cannot carry (ANSI escapes, NUL, lone surrogates).
|
|
23
|
+
|
|
24
|
+
ElementTree serializes them as-is, which leaves a file no parser accepts.
|
|
25
|
+
"""
|
|
26
|
+
return _XML_ILLEGAL.sub("", document)
|
|
27
|
+
|
|
28
|
+
|
|
17
29
|
_BUGTYPE_LABELS = {
|
|
18
30
|
BugType.CONSOLE_ERROR: "Console Error",
|
|
19
31
|
BugType.NETWORK_ERROR: "Network Error",
|
|
@@ -78,10 +90,10 @@ def _repro_badge(receipt: Optional[dict]) -> str:
|
|
|
78
90
|
"#b35900",
|
|
79
91
|
f"NOT REPRODUCED · replayed {receipt.get('steps', '?')} steps, symptom absent",
|
|
80
92
|
)
|
|
81
|
-
elif receipt.get("expect_status") is not None:
|
|
82
|
-
color, label = "#b35900", f"NOT REPRODUCED · expected HTTP {receipt['expect_status']}"
|
|
83
93
|
elif receipt.get("flaky"):
|
|
84
94
|
color, label = "#9a6700", f"INTERMITTENT · {receipt.get('runs', '')} on reload"
|
|
95
|
+
elif receipt.get("expect_status") is not None:
|
|
96
|
+
color, label = "#b35900", f"NOT REPRODUCED · expected HTTP {receipt['expect_status']}"
|
|
85
97
|
else:
|
|
86
98
|
color, label = (
|
|
87
99
|
"#b35900",
|
|
@@ -22,6 +22,13 @@ from .common import (
|
|
|
22
22
|
)
|
|
23
23
|
from .privacy import safe_report
|
|
24
24
|
|
|
25
|
+
# A check with matches=None was inconclusive (login wall, HTTP error, redirect).
|
|
26
|
+
_CHECK_STYLE = {
|
|
27
|
+
True: ("cv-ok", "MATCH"),
|
|
28
|
+
False: ("cv-bad", "MISMATCH"),
|
|
29
|
+
None: ("cv-unknown", "INCONCLUSIVE"),
|
|
30
|
+
}
|
|
31
|
+
|
|
25
32
|
|
|
26
33
|
@safe_report
|
|
27
34
|
def render_html(
|
|
@@ -165,8 +172,8 @@ def render_html(
|
|
|
165
172
|
reference_groups.append(f"<strong>Actions</strong><ul>{values}</ul>")
|
|
166
173
|
if refs.get("verifications"):
|
|
167
174
|
values = "".join(
|
|
168
|
-
f'<li><span class="cv {
|
|
169
|
-
f"{
|
|
175
|
+
f'<li><span class="cv {_CHECK_STYLE[check.get("matches")][0]}">'
|
|
176
|
+
f"{_CHECK_STYLE[check.get('matches')][1]}</span> "
|
|
170
177
|
f"{_esc(check.get('expect', ''))} {_esc(check.get('target_text', ''))} "
|
|
171
178
|
f"@ <code>{_esc(check.get('url', ''))}</code></li>"
|
|
172
179
|
for check in refs["verifications"]
|
|
@@ -307,6 +314,7 @@ h3{{font-size:1.1rem;margin:.5rem 0;color:#f1f5f9}}
|
|
|
307
314
|
.cv{{display:inline-block;padding:1px 5px;border-radius:3px;font-size:.65rem;font-weight:700}}
|
|
308
315
|
.cv-ok{{background:#166534;color:#dcfce7}}
|
|
309
316
|
.cv-bad{{background:#9a3412;color:#ffedd5}}
|
|
317
|
+
.cv-unknown{{background:#475569;color:#f1f5f9}}
|
|
310
318
|
.cs{{display:inline-block;padding:2px 7px;border-radius:4px;font-size:.7rem;font-weight:700;text-transform:uppercase;margin-right:.35rem}}
|
|
311
319
|
.cs-exercised{{background:#166534;color:#dcfce7}}
|
|
312
320
|
.cs-blocked{{background:#9a3412;color:#ffedd5}}
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
from argus.domain.models import ExplorationResult
|
|
4
|
+
from argus.domain.verification import receipt_outcome
|
|
4
5
|
|
|
6
|
+
from .common import xml_safe
|
|
5
7
|
from .privacy import safe_report
|
|
6
8
|
|
|
7
9
|
|
|
@@ -14,9 +16,7 @@ def render_junit(r: ExplorationResult) -> str:
|
|
|
14
16
|
import xml.etree.ElementTree as ET
|
|
15
17
|
|
|
16
18
|
findings = [b.to_dict() for b in r.bugs]
|
|
17
|
-
failures = sum(
|
|
18
|
-
1 for f in findings if (f.get("reproduction") or {}).get("reproduced") is not False
|
|
19
|
-
)
|
|
19
|
+
failures = sum(1 for f in findings if receipt_outcome(f.get("reproduction")) is not False)
|
|
20
20
|
suite = ET.Element(
|
|
21
21
|
"testsuite",
|
|
22
22
|
{
|
|
@@ -28,8 +28,16 @@ def render_junit(r: ExplorationResult) -> str:
|
|
|
28
28
|
},
|
|
29
29
|
)
|
|
30
30
|
for f in findings:
|
|
31
|
-
verdict = (f.get("reproduction")
|
|
32
|
-
tier =
|
|
31
|
+
verdict = receipt_outcome(f.get("reproduction"))
|
|
32
|
+
tier = (
|
|
33
|
+
"PROVEN"
|
|
34
|
+
if verdict is True
|
|
35
|
+
else "REFUTED"
|
|
36
|
+
if verdict is False
|
|
37
|
+
else "INTERMITTENT"
|
|
38
|
+
if (f.get("reproduction") or {}).get("flaky")
|
|
39
|
+
else "UNVERIFIED"
|
|
40
|
+
)
|
|
33
41
|
tc = ET.SubElement(
|
|
34
42
|
suite,
|
|
35
43
|
"testcase",
|
|
@@ -56,4 +64,6 @@ def render_junit(r: ExplorationResult) -> str:
|
|
|
56
64
|
else:
|
|
57
65
|
fail = ET.SubElement(tc, "failure", {"message": f["title"][:200], "type": f["type"]})
|
|
58
66
|
fail.text = "\n".join(body_lines)
|
|
59
|
-
return '<?xml version="1.0" encoding="UTF-8"?>\n' +
|
|
67
|
+
return '<?xml version="1.0" encoding="UTF-8"?>\n' + xml_safe(
|
|
68
|
+
ET.tostring(suite, encoding="unicode")
|
|
69
|
+
)
|
|
@@ -3,6 +3,7 @@ from __future__ import annotations
|
|
|
3
3
|
from typing import Dict
|
|
4
4
|
|
|
5
5
|
from argus.domain.models import ExplorationResult, Severity
|
|
6
|
+
from argus.domain.verification import receipt_outcome
|
|
6
7
|
|
|
7
8
|
from .common import (
|
|
8
9
|
_BUGTYPE_LABELS,
|
|
@@ -14,7 +15,13 @@ from .privacy import safe_report
|
|
|
14
15
|
def render_sarif(r: ExplorationResult) -> str:
|
|
15
16
|
import json
|
|
16
17
|
|
|
17
|
-
_LEVEL = {
|
|
18
|
+
_LEVEL = {
|
|
19
|
+
Severity.CRITICAL: "error",
|
|
20
|
+
Severity.HIGH: "error",
|
|
21
|
+
Severity.MEDIUM: "warning",
|
|
22
|
+
Severity.LOW: "note",
|
|
23
|
+
Severity.INFO: "note",
|
|
24
|
+
}
|
|
18
25
|
rules: Dict[str, dict] = {}
|
|
19
26
|
results = []
|
|
20
27
|
for b in r.bugs:
|
|
@@ -26,7 +33,7 @@ def render_sarif(r: ExplorationResult) -> str:
|
|
|
26
33
|
"name": rid,
|
|
27
34
|
"shortDescription": {"text": _BUGTYPE_LABELS.get(b.type, rid)},
|
|
28
35
|
}
|
|
29
|
-
verdict = (d.get("reproduction")
|
|
36
|
+
verdict = receipt_outcome(d.get("reproduction"))
|
|
30
37
|
msg = d["title"] + (f" — {d['description']}" if d.get("description") else "")
|
|
31
38
|
results.append(
|
|
32
39
|
{
|
|
@@ -41,6 +48,7 @@ def render_sarif(r: ExplorationResult) -> str:
|
|
|
41
48
|
"properties": {
|
|
42
49
|
"verified": d["verified"],
|
|
43
50
|
"reproduced": verdict,
|
|
51
|
+
"intermittent": bool((d.get("reproduction") or {}).get("flaky")),
|
|
44
52
|
"severity": d["severity"],
|
|
45
53
|
},
|
|
46
54
|
}
|