evalview 0.7.1__tar.gz → 0.8.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {evalview-0.7.1/evalview.egg-info → evalview-0.8.0}/PKG-INFO +16 -1
- {evalview-0.7.1 → evalview-0.8.0}/README.md +12 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/cli.py +5 -1
- evalview-0.8.0/evalview/commands/fleet_cmd.py +223 -0
- evalview-0.8.0/evalview/commands/freshness_cmd.py +418 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/monitor_cmd.py +104 -7
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/simulate_cmd.py +112 -3
- evalview-0.8.0/evalview/core/cassette.py +208 -0
- evalview-0.8.0/evalview/core/chaos.py +294 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/config.py +7 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/discord_notifier.py +19 -3
- evalview-0.8.0/evalview/core/fleet.py +442 -0
- evalview-0.8.0/evalview/core/freshness.py +500 -0
- evalview-0.8.0/evalview/core/goal_drift.py +259 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/noise_tracker.py +23 -1
- evalview-0.8.0/evalview/core/otel_semconv.py +307 -0
- evalview-0.8.0/evalview/core/retrieval_lineage.py +318 -0
- evalview-0.8.0/evalview/core/root_cause_hint.py +485 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/simulation.py +208 -6
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/slack_notifier.py +21 -3
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/types.py +38 -0
- {evalview-0.7.1 → evalview-0.8.0/evalview.egg-info}/PKG-INFO +16 -1
- {evalview-0.7.1 → evalview-0.8.0}/evalview.egg-info/SOURCES.txt +18 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview.egg-info/requires.txt +4 -0
- {evalview-0.7.1 → evalview-0.8.0}/pyproject.toml +6 -1
- evalview-0.8.0/tests/test_cassette.py +364 -0
- evalview-0.8.0/tests/test_chaos.py +187 -0
- evalview-0.8.0/tests/test_fleet.py +399 -0
- evalview-0.8.0/tests/test_freshness.py +541 -0
- evalview-0.8.0/tests/test_goal_drift.py +222 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_monitor.py +126 -1
- evalview-0.8.0/tests/test_otel_semconv.py +119 -0
- evalview-0.8.0/tests/test_retrieval_lineage.py +259 -0
- evalview-0.8.0/tests/test_root_cause_hint.py +403 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_simulation.py +96 -0
- {evalview-0.7.1 → evalview-0.8.0}/LICENSE +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/NOTICE +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/__init__.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/__main__.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/adapters/__init__.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/adapters/aider_adapter.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/adapters/anthropic_adapter.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/adapters/base.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/adapters/cohere_adapter.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/adapters/crewai_adapter.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/adapters/crewai_native_adapter.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/adapters/goose_adapter.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/adapters/http_adapter.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/adapters/huggingface_adapter.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/adapters/langgraph_adapter.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/adapters/mcp_adapter.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/adapters/mistral_adapter.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/adapters/ollama_adapter.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/adapters/openai_assistants_adapter.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/adapters/openclaw_adapter.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/adapters/opencode_adapter.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/adapters/pydantic_ai_adapter.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/adapters/registry.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/adapters/tapescope_adapter.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/api.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/benchmarks/__init__.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/benchmarks/canary/README.md +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/benchmarks/canary/__init__.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/benchmarks/canary/suite.v1.held-out.yaml +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/benchmarks/canary/suite.v1.public.yaml +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/chat.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/chat_commands.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/chat_demo.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/chat_prompt.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/chat_runtime.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/chat_session.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/chat_slash.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/ci/__init__.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/ci/comment.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/cloud/__init__.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/cloud/auth.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/cloud/client.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/cloud/push.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/__init__.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/_check_helpers.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/_check_verdict.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/_init_demo_agent.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/_init_tests.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/_init_wizard.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/add_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/assertion_wizard.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/autopr_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/badge_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/baseline_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/benchmark_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/capture_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/chat_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/check_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/check_display.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/ci_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/cloud_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/demo_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/diff_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/drift_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/expand_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/feedback_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/generate_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/golden_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/gym_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/hooks_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/import_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/init_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/judge_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/listing_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/log_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/mcp_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/model_check_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/model_check_render.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/openclaw_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/progress_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/quarantine_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/replay_trace_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/run/__init__.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/run/_adapters.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/run/_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/run/_executor.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/run/_helpers.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/run/_reporter.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/run/_runner.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/shared.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/since_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/skill_agent_runner.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/skill_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/slack_digest_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/snapshot_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/telemetry_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/traces_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/trends_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/validate_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/visual_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/commands/watch_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/compare.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/__init__.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/adapter_factory.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/behavioral_anomalies.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/benchmark_hardening.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/budget.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/canary_suite.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/celebrations.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/contract_diff.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/dashboard.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/diff.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/drift_classifier.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/drift_kind.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/drift_tracker.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/eval_profiles.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/golden.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/healing.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/judge_cache.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/llm_configs.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/llm_provider.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/loader.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/mcp_contract.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/messages.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/model_check_scoring.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/model_provider_runner.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/model_runtime_detector.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/model_snapshots.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/observability.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/parallel.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/pricing.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/project_state.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/quarantine.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/rationale.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/recommendations.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/regression_synth.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/replay_pipeline.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/retry.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/root_cause.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/runner.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/security.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/semantic_diff.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/test_quality.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/tool_categories.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/tracing.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/turn_coherence.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/variant_clusterer.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/verdict.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/core/watcher.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/evaluators/__init__.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/evaluators/cost_evaluator.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/evaluators/evaluator.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/evaluators/hallucination_evaluator.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/evaluators/latency_evaluator.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/evaluators/output_evaluator.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/evaluators/pii_evaluator.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/evaluators/safety_evaluator.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/evaluators/sequence_evaluator.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/evaluators/statistical_evaluator.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/evaluators/tool_call_evaluator.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/expander.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/exporters/__init__.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/exporters/html_exporter.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/importers/__init__.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/importers/log_importer.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/mcp_server.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/openclaw.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/pytest_plugin.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/recorder.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/reporters/__init__.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/reporters/_console_coverage.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/reporters/_console_statistical.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/reporters/_templates.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/reporters/console_reporter.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/reporters/html_reporter.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/reporters/json_reporter.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/reporters/trace_live_reporter.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/reporters/trace_reporter.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/skills/__init__.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/skills/adapters/__init__.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/skills/adapters/base.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/skills/adapters/claude_code_adapter.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/skills/adapters/codex_adapter.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/skills/adapters/crewai_adapter.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/skills/adapters/custom_adapter.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/skills/adapters/langgraph_adapter.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/skills/adapters/openai_assistants_adapter.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/skills/adapters/openclaw_adapter.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/skills/adapters/registry.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/skills/agent_runner.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/skills/agent_types.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/skills/constants.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/skills/evaluators/__init__.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/skills/evaluators/_security_checks.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/skills/evaluators/_security_patterns.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/skills/evaluators/deterministic.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/skills/evaluators/orchestrator.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/skills/evaluators/rubric.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/skills/parser.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/skills/runner.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/skills/security_scanner.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/skills/test_generator.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/skills/test_helpers.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/skills/types.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/skills/ui_utils.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/skills/validator.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/storage/__init__.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/storage/database.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/telemetry/__init__.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/telemetry/client.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/telemetry/config.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/telemetry/decorators.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/telemetry/events.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/templates/patterns/code-generation.yaml +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/templates/patterns/cost-budget.yaml +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/templates/patterns/customer-support.yaml +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/templates/patterns/data-analysis.yaml +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/templates/patterns/latency-budget.yaml +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/templates/patterns/multi-turn-memory.yaml +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/templates/patterns/output-format.yaml +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/templates/patterns/rag-citation.yaml +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/templates/patterns/rag-grounding.yaml +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/templates/patterns/research-synthesis.yaml +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/templates/patterns/safety-refusal.yaml +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/templates/patterns/tool-error-handling.yaml +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/templates/patterns/tool-not-called.yaml +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/templates/patterns/tool-sequence.yaml +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/templates/patterns/wrong-tool-chosen.yaml +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/test_generation.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/test_generation_constants.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/test_generation_discovery.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/test_generation_seeds.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/test_generation_synthesis.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/test_generation_types.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/trace_cmd/__init__.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/trace_cmd/collector.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/trace_cmd/patcher.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/trace_cmd/runner.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/tracking/__init__.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/tracking/database.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/tracking/regression.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/version_check.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/visualization/__init__.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/visualization/_mermaid.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/visualization/_template.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview/visualization/generators.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview.egg-info/dependency_links.txt +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview.egg-info/entry_points.txt +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/evalview.egg-info/top_level.txt +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/setup.cfg +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_adapters.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_api.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_autopr.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_behavioral_anomalies.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_benchmark_hardening.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_canary_suite.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_celebrations.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_check_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_check_pipeline.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_ci_generate_comment.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_config_models.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_diff_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_diff_engine.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_dogfood_e2e.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_drift_classifier.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_drift_tracker.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_e2e_snapshot_check.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_endpoint_detection.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_evaluator_accuracy.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_evaluator_orchestrator.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_evaluators.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_forbidden_tools.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_generate_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_golden_store.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_healing.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_init_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_judge_cache.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_llm_provider.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_loader.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_log_importer_csv.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_main_evaluator.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_mcp_contracts.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_mcp_server.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_messages.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_model_check_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_model_check_scoring.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_model_runtime_detector.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_model_snapshots.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_multi_turn_evaluation.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_new_features.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_noise_tracker.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_observability_regressions.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_pii_evaluator.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_project_state.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_pytest_plugin.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_rationale.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_replay_pipeline.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_root_cause.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_run_cmd_tags.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_run_guidance.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_runner.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_security.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_semantic_diff.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_snapshot_generated_workflow.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_snapshot_json_output.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_statistical_evaluator.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_telemetry_client.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_turn_coherence.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_types.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_validate_cmd.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_verdict.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_version_check.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_visualization_generators.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_week1_integration.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_week2_quarantine.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_week2_review_fixes.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_week2_since_and_comment.py +0 -0
- {evalview-0.7.1 → evalview-0.8.0}/tests/test_week3_progress_and_drift.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: evalview
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.8.0
|
|
4
4
|
Summary: Open-source testing and regression detection framework for AI agents. Golden baseline diffing, CI/CD integration, works with LangGraph, CrewAI, OpenAI, Anthropic Claude, HuggingFace, Ollama, and MCP.
|
|
5
5
|
Author-email: EvalView Team <hidai@evalview.com>
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -41,6 +41,8 @@ Provides-Extra: reports
|
|
|
41
41
|
Requires-Dist: plotly>=5.0; extra == "reports"
|
|
42
42
|
Provides-Extra: watch
|
|
43
43
|
Requires-Dist: watchdog>=3.0; extra == "watch"
|
|
44
|
+
Provides-Extra: schedule
|
|
45
|
+
Requires-Dist: croniter>=2.0; extra == "schedule"
|
|
44
46
|
Provides-Extra: telemetry
|
|
45
47
|
Requires-Dist: posthog>=3.0.0; extra == "telemetry"
|
|
46
48
|
Provides-Extra: cohere
|
|
@@ -50,6 +52,7 @@ Requires-Dist: mistralai>=1.0.0; extra == "mistral"
|
|
|
50
52
|
Provides-Extra: all
|
|
51
53
|
Requires-Dist: plotly>=5.0; extra == "all"
|
|
52
54
|
Requires-Dist: watchdog>=3.0; extra == "all"
|
|
55
|
+
Requires-Dist: croniter>=2.0; extra == "all"
|
|
53
56
|
Requires-Dist: posthog>=3.0.0; extra == "all"
|
|
54
57
|
Requires-Dist: cohere>=5.0.0; extra == "all"
|
|
55
58
|
Requires-Dist: mistralai>=1.0.0; extra == "all"
|
|
@@ -106,6 +109,7 @@ Most eval tools stop at detect and compare. EvalView helps you classify changes,
|
|
|
106
109
|
- Catch silent regressions that normal tests miss
|
|
107
110
|
- Separate provider/model drift from real system regressions
|
|
108
111
|
- Auto-heal flaky failures with retries, review gates, and audit logs
|
|
112
|
+
- Replay deterministically — cassettes capture real tool calls once so CI never re-hits live services
|
|
109
113
|
|
|
110
114
|
Built for **frontier-lab rigor, startup-team practicality**:
|
|
111
115
|
- targeted behavior runs instead of giant always-on eval suites
|
|
@@ -340,6 +344,7 @@ Use LangSmith for observability. Use Braintrust for scoring. **Use EvalView for
|
|
|
340
344
|
| Golden baseline regression | — | Manual | — | **Automatic** |
|
|
341
345
|
| Silent model change detection | — | — | — | **Yes** |
|
|
342
346
|
| Auto-heal (retry + variant proposal) | — | — | — | **Yes** |
|
|
347
|
+
| Hermetic record/replay | — | — | — | **Yes** |
|
|
343
348
|
| PR comments with alerts | — | — | — | **Cost, latency, model change** |
|
|
344
349
|
| Works without API keys | No | No | Partial | **Yes** |
|
|
345
350
|
| Production monitoring | Tracing | — | — | **Check loop + Slack** |
|
|
@@ -775,6 +780,15 @@ Two features that close the gaps the April 2026 agent-eval reports kept calling
|
|
|
775
780
|
evalview simulate tests/ --variants 5 --seed 42
|
|
776
781
|
```
|
|
777
782
|
|
|
783
|
+
**Record/replay cassettes** — declarative mocks are tedious for agents with dozens of tools. Record once against the real backend, replay forever — no network, no LLM cost beyond the agent's own model calls, no flaky third-party APIs in CI. *"The agent saw X, called Y, got Z"* becomes deterministic without re-hitting live services.
|
|
784
|
+
|
|
785
|
+
```bash
|
|
786
|
+
evalview simulate tests/refund.yaml --record # capture real tool results once
|
|
787
|
+
evalview simulate tests/refund.yaml --replay # hermetic from now on
|
|
788
|
+
```
|
|
789
|
+
|
|
790
|
+
Cassettes live at `.evalview/cassettes/<test>.json`, use per-tool sequential matching (robust to inter-tool ordering drift), and are versioned for forward compatibility. Declarative `mocks:` still take precedence so you can override a single recording without re-recording the whole run.
|
|
791
|
+
|
|
778
792
|
**Decision rationale** — every tool_choice / branch gets recorded with the chosen option, alternatives considered, and any model-reported reasoning (Anthropic `thinking` blocks auto-captured). Grouped across runs by `input_hash` so cloud analytics surfaces decision drift before your users notice. Local HTML replay shows it inline.
|
|
779
793
|
|
|
780
794
|
Supported adapters: Anthropic, OpenAI Assistants, LangGraph, CrewAI native.
|
|
@@ -793,6 +807,7 @@ Supported adapters: Anthropic, OpenAI Assistants, LangGraph, CrewAI native.
|
|
|
793
807
|
| **Recommendation engine** | Suggests the next command from verdict, drift class, and history | [Above](#daily-workflow) |
|
|
794
808
|
| **Model drift detection** | `model-check` — zero-judge canary suite that catches silent model updates | [Docs](#model-drift-detection) |
|
|
795
809
|
| **Simulation harness** | `evalview simulate` — hermetic what-if runs against declared mocks, with `--variants N` fan-out | [Docs](docs/SIMULATE.md) |
|
|
810
|
+
| **Record/replay cassettes** | Capture real tool calls once, replay deterministically forever — no live services in CI | [Docs](docs/SIMULATE.md#record--replay-cassettes) |
|
|
796
811
|
| **Decision rationale** | Structured `tool_choice` / `branch` logging with cross-run grouping for decision-drift detection | [Docs](docs/RATIONALE.md) |
|
|
797
812
|
| **Assertion wizard** | Analyze captured traffic, suggest smart assertions automatically | [Above](#assertion-wizard--tests-from-real-traffic) |
|
|
798
813
|
| **Auto-variant discovery** | Run N times, cluster paths, save valid variants | [Above](#auto-variant-discovery--solve-non-determinism) |
|
|
@@ -35,6 +35,7 @@ Most eval tools stop at detect and compare. EvalView helps you classify changes,
|
|
|
35
35
|
- Catch silent regressions that normal tests miss
|
|
36
36
|
- Separate provider/model drift from real system regressions
|
|
37
37
|
- Auto-heal flaky failures with retries, review gates, and audit logs
|
|
38
|
+
- Replay deterministically — cassettes capture real tool calls once so CI never re-hits live services
|
|
38
39
|
|
|
39
40
|
Built for **frontier-lab rigor, startup-team practicality**:
|
|
40
41
|
- targeted behavior runs instead of giant always-on eval suites
|
|
@@ -269,6 +270,7 @@ Use LangSmith for observability. Use Braintrust for scoring. **Use EvalView for
|
|
|
269
270
|
| Golden baseline regression | — | Manual | — | **Automatic** |
|
|
270
271
|
| Silent model change detection | — | — | — | **Yes** |
|
|
271
272
|
| Auto-heal (retry + variant proposal) | — | — | — | **Yes** |
|
|
273
|
+
| Hermetic record/replay | — | — | — | **Yes** |
|
|
272
274
|
| PR comments with alerts | — | — | — | **Cost, latency, model change** |
|
|
273
275
|
| Works without API keys | No | No | Partial | **Yes** |
|
|
274
276
|
| Production monitoring | Tracing | — | — | **Check loop + Slack** |
|
|
@@ -704,6 +706,15 @@ Two features that close the gaps the April 2026 agent-eval reports kept calling
|
|
|
704
706
|
evalview simulate tests/ --variants 5 --seed 42
|
|
705
707
|
```
|
|
706
708
|
|
|
709
|
+
**Record/replay cassettes** — declarative mocks are tedious for agents with dozens of tools. Record once against the real backend, replay forever — no network, no LLM cost beyond the agent's own model calls, no flaky third-party APIs in CI. *"The agent saw X, called Y, got Z"* becomes deterministic without re-hitting live services.
|
|
710
|
+
|
|
711
|
+
```bash
|
|
712
|
+
evalview simulate tests/refund.yaml --record # capture real tool results once
|
|
713
|
+
evalview simulate tests/refund.yaml --replay # hermetic from now on
|
|
714
|
+
```
|
|
715
|
+
|
|
716
|
+
Cassettes live at `.evalview/cassettes/<test>.json`, use per-tool sequential matching (robust to inter-tool ordering drift), and are versioned for forward compatibility. Declarative `mocks:` still take precedence so you can override a single recording without re-recording the whole run.
|
|
717
|
+
|
|
707
718
|
**Decision rationale** — every tool_choice / branch gets recorded with the chosen option, alternatives considered, and any model-reported reasoning (Anthropic `thinking` blocks auto-captured). Grouped across runs by `input_hash` so cloud analytics surfaces decision drift before your users notice. Local HTML replay shows it inline.
|
|
708
719
|
|
|
709
720
|
Supported adapters: Anthropic, OpenAI Assistants, LangGraph, CrewAI native.
|
|
@@ -722,6 +733,7 @@ Supported adapters: Anthropic, OpenAI Assistants, LangGraph, CrewAI native.
|
|
|
722
733
|
| **Recommendation engine** | Suggests the next command from verdict, drift class, and history | [Above](#daily-workflow) |
|
|
723
734
|
| **Model drift detection** | `model-check` — zero-judge canary suite that catches silent model updates | [Docs](#model-drift-detection) |
|
|
724
735
|
| **Simulation harness** | `evalview simulate` — hermetic what-if runs against declared mocks, with `--variants N` fan-out | [Docs](docs/SIMULATE.md) |
|
|
736
|
+
| **Record/replay cassettes** | Capture real tool calls once, replay deterministically forever — no live services in CI | [Docs](docs/SIMULATE.md#record--replay-cassettes) |
|
|
725
737
|
| **Decision rationale** | Structured `tool_choice` / `branch` logging with cross-run grouping for decision-drift detection | [Docs](docs/RATIONALE.md) |
|
|
726
738
|
| **Assertion wizard** | Analyze captured traffic, suggest smart assertions automatically | [Above](#assertion-wizard--tests-from-real-traffic) |
|
|
727
739
|
| **Auto-variant discovery** | Run N times, cluster paths, save valid variants | [Above](#auto-variant-discovery--solve-non-determinism) |
|
|
@@ -55,6 +55,8 @@ from evalview.commands.replay_trace_cmd import replay_trace
|
|
|
55
55
|
from evalview.commands.model_check_cmd import model_check
|
|
56
56
|
from evalview.commands.monitor_cmd import monitor
|
|
57
57
|
from evalview.commands.autopr_cmd import autopr
|
|
58
|
+
from evalview.commands.freshness_cmd import freshness
|
|
59
|
+
from evalview.commands.fleet_cmd import fleet_cmd
|
|
58
60
|
from evalview.commands.benchmark_cmd import benchmark_cmd
|
|
59
61
|
from evalview.commands.mcp_cmd import mcp
|
|
60
62
|
from evalview.commands.validate_cmd import validate
|
|
@@ -87,7 +89,7 @@ class OrderedGroup(click.Group):
|
|
|
87
89
|
("Start", ["demo", "init"]),
|
|
88
90
|
("Regression Gating", ["snapshot", "check", "model-check", "replay"]),
|
|
89
91
|
("Triage", ["since", "progress", "drift"]),
|
|
90
|
-
("Production", ["watch", "monitor", "autopr"]),
|
|
92
|
+
("Production", ["watch", "monitor", "autopr", "freshness", "fleet"]),
|
|
91
93
|
("Evaluation", ["run", "report", "generate", "judge", "expand",
|
|
92
94
|
"golden", "compare", "benchmark", "simulate"]),
|
|
93
95
|
("Capture & Import", ["capture", "import", "record", "connect", "adapters"]),
|
|
@@ -214,6 +216,8 @@ main.add_command(traces)
|
|
|
214
216
|
main.add_command(baseline)
|
|
215
217
|
main.add_command(monitor)
|
|
216
218
|
main.add_command(autopr)
|
|
219
|
+
main.add_command(freshness)
|
|
220
|
+
main.add_command(fleet_cmd, name="fleet")
|
|
217
221
|
main.add_command(feedback)
|
|
218
222
|
main.add_command(openclaw)
|
|
219
223
|
main.add_command(watch)
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
"""`evalview fleet` — cross-instance rollup of monitor history files.
|
|
2
|
+
|
|
3
|
+
A canary instance, a prod instance, a dev's laptop, three regional pods —
|
|
4
|
+
they each write their own ``monitor --history`` JSONL. The single-history
|
|
5
|
+
commands (``since``, ``progress``, ``drift``) tell you what happened in
|
|
6
|
+
one of them. ``fleet`` tells you what's happening across all of them.
|
|
7
|
+
|
|
8
|
+
evalview fleet --dir .evalview/history/
|
|
9
|
+
evalview fleet --history monitor-eu.jsonl --history monitor-us.jsonl
|
|
10
|
+
evalview fleet --json > fleet.json
|
|
11
|
+
evalview fleet --anomalies-only # only show pods deviating from mean
|
|
12
|
+
evalview fleet --require-clean # CI gate
|
|
13
|
+
|
|
14
|
+
Pure analytics. No network, no LLM.
|
|
15
|
+
"""
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import json
|
|
19
|
+
import sys
|
|
20
|
+
from typing import Tuple
|
|
21
|
+
|
|
22
|
+
import click
|
|
23
|
+
|
|
24
|
+
from evalview.commands.shared import console
|
|
25
|
+
from evalview.core.fleet import (
|
|
26
|
+
DEFAULT_ANOMALY_SIGMA,
|
|
27
|
+
DEFAULT_TEST_IMPACT_PCT,
|
|
28
|
+
FleetReport,
|
|
29
|
+
build_fleet_report,
|
|
30
|
+
discover_history_files,
|
|
31
|
+
)
|
|
32
|
+
from evalview.telemetry.decorators import track_command
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _render(report: FleetReport, anomalies_only: bool) -> None:
|
|
36
|
+
"""Pretty-print the report to the console.
|
|
37
|
+
|
|
38
|
+
Layout mirrors the slack-digest aesthetic: one hero number, one
|
|
39
|
+
concern, one action. We keep individual instance rows short so the
|
|
40
|
+
table fits in a terminal even with 20+ pods.
|
|
41
|
+
"""
|
|
42
|
+
n = len(report.instances)
|
|
43
|
+
if n == 0:
|
|
44
|
+
console.print("[dim]No history files matched. Pass --history FILE or "
|
|
45
|
+
"--dir DIR pointing at JSONLs written by "
|
|
46
|
+
"`evalview monitor --history`.[/dim]")
|
|
47
|
+
return
|
|
48
|
+
|
|
49
|
+
console.print()
|
|
50
|
+
console.print("[bold]EvalView Fleet[/bold]")
|
|
51
|
+
pct = report.fleet_pass_rate * 100
|
|
52
|
+
color = "green" if pct >= 95 else "yellow" if pct >= 80 else "red"
|
|
53
|
+
console.print(
|
|
54
|
+
f" Instances: [cyan]{n}[/cyan] "
|
|
55
|
+
f"| Cycles: [cyan]{report.fleet_cycles}[/cyan] "
|
|
56
|
+
f"| Pass rate: [{color}]{pct:.1f}%[/{color}] "
|
|
57
|
+
f"| Cost: [cyan]${report.fleet_cost:.4f}[/cyan]"
|
|
58
|
+
)
|
|
59
|
+
if report.fleet_regressions:
|
|
60
|
+
console.print(
|
|
61
|
+
f" [red]Regressions across fleet: {report.fleet_regressions}[/red]"
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
# Anomalies first — these are the "stop the line" rows. Surface them
|
|
65
|
+
# *before* the full table so they don't get lost in a 20-pod fleet.
|
|
66
|
+
if report.anomalies:
|
|
67
|
+
console.print()
|
|
68
|
+
console.print("[bold yellow]Anomalies[/bold yellow] "
|
|
69
|
+
f"[dim](≥ {report.anomaly_sigma:.1f}σ from fleet mean)[/dim]")
|
|
70
|
+
for a in report.anomalies:
|
|
71
|
+
arrow = "↓" if a.direction == "below" else "↑"
|
|
72
|
+
console.print(
|
|
73
|
+
f" {arrow} [cyan]{a.instance}[/cyan] "
|
|
74
|
+
f"pass {a.pass_rate * 100:.1f}% "
|
|
75
|
+
f"({a.sigma_distance:+.1f}σ vs fleet mean {a.fleet_mean * 100:.1f}%)"
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
if anomalies_only:
|
|
79
|
+
return
|
|
80
|
+
|
|
81
|
+
# Fleet-wide failures: a test failing across many pods means fixing
|
|
82
|
+
# one pod won't help. This is the "is it everywhere?" callout.
|
|
83
|
+
if report.fleet_wide_failures:
|
|
84
|
+
console.print()
|
|
85
|
+
console.print(
|
|
86
|
+
"[bold]Fleet-wide failing tests[/bold] "
|
|
87
|
+
f"[dim](≥ {report.test_impact_pct * 100:.0f}% of instances)[/dim]"
|
|
88
|
+
)
|
|
89
|
+
for f in report.fleet_wide_failures:
|
|
90
|
+
console.print(
|
|
91
|
+
f" • [red]{f.test_name}[/red] "
|
|
92
|
+
f"({f.impact_pct * 100:.0f}% — "
|
|
93
|
+
f"{', '.join(f.affected_instances[:4])}"
|
|
94
|
+
f"{', …' if len(f.affected_instances) > 4 else ''})"
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
console.print()
|
|
98
|
+
console.print("[bold]Per-instance[/bold]")
|
|
99
|
+
# Sort: anomalies bubble up via the report's own pass_rate sort.
|
|
100
|
+
for s in report.instances:
|
|
101
|
+
rate = s.pass_rate * 100
|
|
102
|
+
bar = "▇" * min(int(rate / 10), 10) + "·" * max(0, 10 - int(rate / 10))
|
|
103
|
+
color = "green" if rate >= 95 else "yellow" if rate >= 80 else "red"
|
|
104
|
+
failing_part = (
|
|
105
|
+
f" [dim]{len(s.failing_tests)} failing[/dim]" if s.failing_tests else ""
|
|
106
|
+
)
|
|
107
|
+
console.print(
|
|
108
|
+
f" [{color}]{bar}[/{color}] "
|
|
109
|
+
f"[cyan]{s.instance:<20.20}[/cyan] "
|
|
110
|
+
f"{rate:5.1f}% "
|
|
111
|
+
f"[dim]{s.cycles:>4} cyc[/dim] "
|
|
112
|
+
f"[dim]${s.cost:.4f}[/dim]"
|
|
113
|
+
f"{failing_part}"
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _verdict(report: FleetReport) -> Tuple[str, int]:
|
|
118
|
+
"""Return ``(label, exit_code)`` for ``--require-clean``.
|
|
119
|
+
|
|
120
|
+
Clean = no anomalies, no fleet-wide failures, no regressions. We're
|
|
121
|
+
intentionally strict here: ``--require-clean`` is for the CI gate
|
|
122
|
+
use case where any cross-instance signal should block.
|
|
123
|
+
"""
|
|
124
|
+
if report.fleet_regressions or report.anomalies or report.fleet_wide_failures:
|
|
125
|
+
return "fleet has cross-instance signal — investigate", 1
|
|
126
|
+
return "fleet is clean across all instances", 0
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
@click.command("fleet")
|
|
130
|
+
@click.option(
|
|
131
|
+
"--history",
|
|
132
|
+
"history_paths",
|
|
133
|
+
multiple=True,
|
|
134
|
+
type=click.Path(),
|
|
135
|
+
help="Path to a monitor history JSONL. Repeat for multiple instances. "
|
|
136
|
+
"Globs are expanded. Combine with --dir.",
|
|
137
|
+
)
|
|
138
|
+
@click.option(
|
|
139
|
+
"--dir",
|
|
140
|
+
"directories",
|
|
141
|
+
multiple=True,
|
|
142
|
+
type=click.Path(),
|
|
143
|
+
help="Scan this directory for *.jsonl files (non-recursive). "
|
|
144
|
+
"Repeat for multiple roots.",
|
|
145
|
+
)
|
|
146
|
+
@click.option(
|
|
147
|
+
"--anomaly-sigma",
|
|
148
|
+
type=float,
|
|
149
|
+
default=DEFAULT_ANOMALY_SIGMA,
|
|
150
|
+
show_default=True,
|
|
151
|
+
help="Z-score threshold for flagging an instance as anomalous.",
|
|
152
|
+
)
|
|
153
|
+
@click.option(
|
|
154
|
+
"--test-impact-pct",
|
|
155
|
+
type=float,
|
|
156
|
+
default=DEFAULT_TEST_IMPACT_PCT,
|
|
157
|
+
show_default=True,
|
|
158
|
+
help="Fraction of instances a test must fail in to count as fleet-wide.",
|
|
159
|
+
)
|
|
160
|
+
@click.option(
|
|
161
|
+
"--anomalies-only",
|
|
162
|
+
is_flag=True,
|
|
163
|
+
help="Skip the per-instance and fleet-wide tables; print only anomalies.",
|
|
164
|
+
)
|
|
165
|
+
@click.option(
|
|
166
|
+
"--require-clean",
|
|
167
|
+
is_flag=True,
|
|
168
|
+
help="Exit 1 if the fleet has any regression, anomaly, or fleet-wide "
|
|
169
|
+
"failure. Useful in CI to block deploys when canary diverges.",
|
|
170
|
+
)
|
|
171
|
+
@click.option(
|
|
172
|
+
"--json",
|
|
173
|
+
"json_output",
|
|
174
|
+
is_flag=True,
|
|
175
|
+
help="Emit a machine-readable JSON report to stdout.",
|
|
176
|
+
)
|
|
177
|
+
@track_command("fleet")
|
|
178
|
+
def fleet_cmd(
|
|
179
|
+
history_paths: Tuple[str, ...],
|
|
180
|
+
directories: Tuple[str, ...],
|
|
181
|
+
anomaly_sigma: float,
|
|
182
|
+
test_impact_pct: float,
|
|
183
|
+
anomalies_only: bool,
|
|
184
|
+
require_clean: bool,
|
|
185
|
+
json_output: bool,
|
|
186
|
+
) -> None:
|
|
187
|
+
"""Roll up multiple monitor history files into one fleet view.
|
|
188
|
+
|
|
189
|
+
Each ``--history FILE`` (or each ``*.jsonl`` in a ``--dir``) is one
|
|
190
|
+
monitor session — typically one agent instance or one region. The
|
|
191
|
+
command produces a fleet summary, per-instance breakdown, anomaly
|
|
192
|
+
callouts (instances ≥ 2σ off the fleet mean), and fleet-wide
|
|
193
|
+
failures (tests failing in ≥ 40% of instances).
|
|
194
|
+
|
|
195
|
+
\b
|
|
196
|
+
Examples:
|
|
197
|
+
evalview fleet --dir .evalview/history/
|
|
198
|
+
evalview fleet --history monitor-eu.jsonl --history monitor-us.jsonl
|
|
199
|
+
evalview fleet --json > fleet.json
|
|
200
|
+
evalview fleet --anomalies-only
|
|
201
|
+
evalview fleet --require-clean # CI gate
|
|
202
|
+
|
|
203
|
+
History files are produced by `evalview monitor --history FILE.jsonl`.
|
|
204
|
+
"""
|
|
205
|
+
files = discover_history_files(history_paths, directories)
|
|
206
|
+
report = build_fleet_report(
|
|
207
|
+
files,
|
|
208
|
+
anomaly_sigma=anomaly_sigma,
|
|
209
|
+
test_impact_pct=test_impact_pct,
|
|
210
|
+
)
|
|
211
|
+
|
|
212
|
+
if json_output:
|
|
213
|
+
click.echo(json.dumps(report.to_dict(), indent=2))
|
|
214
|
+
else:
|
|
215
|
+
_render(report, anomalies_only=anomalies_only)
|
|
216
|
+
|
|
217
|
+
if require_clean:
|
|
218
|
+
label, code = _verdict(report)
|
|
219
|
+
if not json_output:
|
|
220
|
+
color = "green" if code == 0 else "red"
|
|
221
|
+
console.print()
|
|
222
|
+
console.print(f" [{color}]{label}[/{color}]")
|
|
223
|
+
sys.exit(code)
|