evalview 0.6.0__tar.gz → 0.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {evalview-0.6.0/evalview.egg-info → evalview-0.7.0}/PKG-INFO +373 -12
- {evalview-0.6.0 → evalview-0.7.0}/README.md +371 -11
- evalview-0.7.0/evalview/__init__.py +15 -0
- evalview-0.7.0/evalview/adapters/aider_adapter.py +535 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/adapters/anthropic_adapter.py +33 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/adapters/crewai_native_adapter.py +34 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/adapters/langgraph_adapter.py +47 -2
- {evalview-0.6.0 → evalview-0.7.0}/evalview/adapters/openai_assistants_adapter.py +18 -0
- evalview-0.7.0/evalview/adapters/opencode_adapter.py +387 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/adapters/registry.py +14 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/api.py +13 -1
- evalview-0.7.0/evalview/benchmarks/canary/README.md +93 -0
- evalview-0.7.0/evalview/benchmarks/canary/__init__.py +18 -0
- evalview-0.7.0/evalview/benchmarks/canary/suite.v1.held-out.yaml +66 -0
- evalview-0.7.0/evalview/benchmarks/canary/suite.v1.public.yaml +214 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/chat.py +23 -7
- {evalview-0.6.0 → evalview-0.7.0}/evalview/ci/comment.py +255 -2
- {evalview-0.6.0 → evalview-0.7.0}/evalview/cli.py +37 -1
- evalview-0.7.0/evalview/cloud/push.py +328 -0
- evalview-0.7.0/evalview/commands/autopr_cmd.py +526 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/check_cmd.py +540 -14
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/check_display.py +358 -9
- evalview-0.7.0/evalview/commands/drift_cmd.py +335 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/listing_cmd.py +4 -6
- evalview-0.7.0/evalview/commands/log_cmd.py +212 -0
- evalview-0.7.0/evalview/commands/model_check_cmd.py +691 -0
- evalview-0.7.0/evalview/commands/model_check_render.py +149 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/monitor_cmd.py +406 -115
- evalview-0.7.0/evalview/commands/progress_cmd.py +413 -0
- evalview-0.7.0/evalview/commands/quarantine_cmd.py +191 -0
- evalview-0.7.0/evalview/commands/replay_trace_cmd.py +257 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/run/_adapters.py +40 -2
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/run/_cmd.py +58 -5
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/run/_reporter.py +63 -5
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/shared.py +122 -39
- evalview-0.7.0/evalview/commands/simulate_cmd.py +213 -0
- evalview-0.7.0/evalview/commands/since_cmd.py +536 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/skill_cmd.py +20 -2
- evalview-0.7.0/evalview/commands/slack_digest_cmd.py +423 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/snapshot_cmd.py +133 -59
- evalview-0.7.0/evalview/compare.py +486 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/core/adapter_factory.py +17 -1
- evalview-0.7.0/evalview/core/behavioral_anomalies.py +478 -0
- evalview-0.7.0/evalview/core/benchmark_hardening.py +550 -0
- evalview-0.7.0/evalview/core/canary_suite.py +171 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/core/celebrations.py +3 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/core/config.py +17 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/core/diff.py +8 -0
- evalview-0.7.0/evalview/core/discord_notifier.py +139 -0
- evalview-0.7.0/evalview/core/drift_classifier.py +205 -0
- evalview-0.7.0/evalview/core/drift_kind.py +49 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/core/drift_tracker.py +168 -1
- evalview-0.7.0/evalview/core/model_check_scoring.py +389 -0
- evalview-0.7.0/evalview/core/model_provider_runner.py +224 -0
- evalview-0.7.0/evalview/core/model_snapshots.py +321 -0
- evalview-0.7.0/evalview/core/noise_tracker.py +529 -0
- evalview-0.7.0/evalview/core/observability.py +233 -0
- evalview-0.7.0/evalview/core/quarantine.py +451 -0
- evalview-0.7.0/evalview/core/rationale.py +226 -0
- evalview-0.7.0/evalview/core/recommendations.py +337 -0
- evalview-0.7.0/evalview/core/regression_synth.py +271 -0
- evalview-0.7.0/evalview/core/replay_pipeline.py +330 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/core/root_cause.py +243 -8
- evalview-0.7.0/evalview/core/simulation.py +337 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/core/slack_notifier.py +43 -5
- evalview-0.7.0/evalview/core/turn_coherence.py +567 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/core/types.py +283 -1
- evalview-0.7.0/evalview/core/verdict.py +242 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/evaluators/cost_evaluator.py +7 -2
- {evalview-0.6.0 → evalview-0.7.0}/evalview/evaluators/evaluator.py +35 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/mcp_server.py +260 -6
- {evalview-0.6.0 → evalview-0.7.0}/evalview/pytest_plugin.py +38 -1
- {evalview-0.6.0 → evalview-0.7.0}/evalview/reporters/console_reporter.py +5 -1
- {evalview-0.6.0 → evalview-0.7.0}/evalview/reporters/html_reporter.py +293 -10
- {evalview-0.6.0 → evalview-0.7.0}/evalview/skills/types.py +15 -1
- evalview-0.7.0/evalview/templates/patterns/code-generation.yaml +31 -0
- evalview-0.7.0/evalview/templates/patterns/cost-budget.yaml +20 -0
- evalview-0.7.0/evalview/templates/patterns/customer-support.yaml +31 -0
- evalview-0.7.0/evalview/templates/patterns/data-analysis.yaml +31 -0
- evalview-0.7.0/evalview/templates/patterns/latency-budget.yaml +20 -0
- evalview-0.7.0/evalview/templates/patterns/multi-turn-memory.yaml +28 -0
- evalview-0.7.0/evalview/templates/patterns/output-format.yaml +23 -0
- evalview-0.7.0/evalview/templates/patterns/rag-citation.yaml +29 -0
- evalview-0.7.0/evalview/templates/patterns/rag-grounding.yaml +27 -0
- evalview-0.7.0/evalview/templates/patterns/research-synthesis.yaml +34 -0
- evalview-0.7.0/evalview/templates/patterns/safety-refusal.yaml +28 -0
- evalview-0.7.0/evalview/templates/patterns/tool-error-handling.yaml +26 -0
- evalview-0.7.0/evalview/templates/patterns/tool-not-called.yaml +22 -0
- evalview-0.7.0/evalview/templates/patterns/tool-sequence.yaml +26 -0
- evalview-0.7.0/evalview/templates/patterns/wrong-tool-chosen.yaml +23 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/test_generation.py +5 -1
- {evalview-0.6.0 → evalview-0.7.0}/evalview/visualization/generators.py +380 -4
- {evalview-0.6.0 → evalview-0.7.0/evalview.egg-info}/PKG-INFO +373 -12
- {evalview-0.6.0 → evalview-0.7.0}/evalview.egg-info/SOURCES.txt +77 -1
- {evalview-0.6.0 → evalview-0.7.0}/evalview.egg-info/requires.txt +1 -0
- {evalview-0.6.0 → evalview-0.7.0}/pyproject.toml +9 -1
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_adapters.py +80 -0
- evalview-0.7.0/tests/test_autopr.py +475 -0
- evalview-0.7.0/tests/test_behavioral_anomalies.py +457 -0
- evalview-0.7.0/tests/test_benchmark_hardening.py +484 -0
- evalview-0.7.0/tests/test_canary_suite.py +171 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_check_cmd.py +383 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_check_pipeline.py +81 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_config_models.py +1 -0
- evalview-0.7.0/tests/test_drift_classifier.py +148 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_evaluator_accuracy.py +2 -2
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_loader.py +43 -0
- evalview-0.7.0/tests/test_mcp_server.py +489 -0
- evalview-0.7.0/tests/test_model_check_cmd.py +796 -0
- evalview-0.7.0/tests/test_model_check_scoring.py +279 -0
- evalview-0.7.0/tests/test_model_snapshots.py +272 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_monitor.py +176 -1
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_new_features.py +10 -0
- evalview-0.7.0/tests/test_noise_tracker.py +449 -0
- evalview-0.7.0/tests/test_observability_regressions.py +330 -0
- evalview-0.7.0/tests/test_rationale.py +317 -0
- evalview-0.7.0/tests/test_replay_pipeline.py +327 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_root_cause.py +381 -22
- evalview-0.7.0/tests/test_run_cmd_tags.py +44 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_runner.py +8 -2
- evalview-0.7.0/tests/test_simulation.py +359 -0
- evalview-0.7.0/tests/test_snapshot_json_output.py +247 -0
- evalview-0.7.0/tests/test_turn_coherence.py +409 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_types.py +261 -0
- evalview-0.7.0/tests/test_verdict.py +228 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_visualization_generators.py +70 -0
- evalview-0.7.0/tests/test_week1_integration.py +330 -0
- evalview-0.7.0/tests/test_week2_quarantine.py +222 -0
- evalview-0.7.0/tests/test_week2_review_fixes.py +284 -0
- evalview-0.7.0/tests/test_week2_since_and_comment.py +344 -0
- evalview-0.7.0/tests/test_week3_progress_and_drift.py +610 -0
- evalview-0.6.0/evalview/__init__.py +0 -6
- {evalview-0.6.0 → evalview-0.7.0}/LICENSE +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/NOTICE +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/__main__.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/adapters/__init__.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/adapters/base.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/adapters/cohere_adapter.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/adapters/crewai_adapter.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/adapters/goose_adapter.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/adapters/http_adapter.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/adapters/huggingface_adapter.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/adapters/mcp_adapter.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/adapters/mistral_adapter.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/adapters/ollama_adapter.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/adapters/openclaw_adapter.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/adapters/pydantic_ai_adapter.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/adapters/tapescope_adapter.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/benchmarks/__init__.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/chat_runtime.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/chat_slash.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/ci/__init__.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/cloud/__init__.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/cloud/auth.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/cloud/client.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/__init__.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/add_cmd.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/assertion_wizard.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/badge_cmd.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/baseline_cmd.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/benchmark_cmd.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/capture_cmd.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/chat_cmd.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/ci_cmd.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/cloud_cmd.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/demo_cmd.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/expand_cmd.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/feedback_cmd.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/generate_cmd.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/golden_cmd.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/gym_cmd.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/hooks_cmd.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/import_cmd.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/init_cmd.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/judge_cmd.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/mcp_cmd.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/openclaw_cmd.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/run/__init__.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/run/_executor.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/run/_runner.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/skill_agent_runner.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/telemetry_cmd.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/traces_cmd.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/trends_cmd.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/visual_cmd.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/commands/watch_cmd.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/core/__init__.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/core/budget.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/core/contract_diff.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/core/dashboard.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/core/eval_profiles.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/core/golden.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/core/healing.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/core/judge_cache.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/core/llm_configs.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/core/llm_provider.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/core/loader.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/core/mcp_contract.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/core/messages.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/core/model_runtime_detector.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/core/parallel.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/core/pricing.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/core/project_state.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/core/retry.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/core/runner.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/core/security.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/core/semantic_diff.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/core/test_quality.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/core/tool_categories.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/core/tracing.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/core/variant_clusterer.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/core/watcher.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/evaluators/__init__.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/evaluators/hallucination_evaluator.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/evaluators/latency_evaluator.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/evaluators/output_evaluator.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/evaluators/pii_evaluator.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/evaluators/safety_evaluator.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/evaluators/sequence_evaluator.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/evaluators/statistical_evaluator.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/evaluators/tool_call_evaluator.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/expander.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/exporters/__init__.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/exporters/html_exporter.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/importers/__init__.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/importers/log_importer.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/openclaw.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/recorder.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/reporters/__init__.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/reporters/json_reporter.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/reporters/trace_live_reporter.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/reporters/trace_reporter.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/skills/__init__.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/skills/adapters/__init__.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/skills/adapters/base.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/skills/adapters/claude_code_adapter.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/skills/adapters/codex_adapter.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/skills/adapters/crewai_adapter.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/skills/adapters/custom_adapter.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/skills/adapters/langgraph_adapter.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/skills/adapters/openai_assistants_adapter.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/skills/adapters/openclaw_adapter.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/skills/adapters/registry.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/skills/agent_runner.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/skills/agent_types.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/skills/constants.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/skills/evaluators/__init__.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/skills/evaluators/deterministic.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/skills/evaluators/orchestrator.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/skills/evaluators/rubric.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/skills/parser.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/skills/runner.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/skills/security_scanner.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/skills/test_generator.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/skills/test_helpers.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/skills/ui_utils.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/skills/validator.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/storage/__init__.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/storage/database.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/telemetry/__init__.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/telemetry/client.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/telemetry/config.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/telemetry/decorators.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/telemetry/events.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/trace_cmd/__init__.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/trace_cmd/collector.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/trace_cmd/patcher.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/trace_cmd/runner.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/tracking/__init__.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/tracking/database.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/tracking/regression.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/version_check.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview/visualization/__init__.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview.egg-info/dependency_links.txt +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview.egg-info/entry_points.txt +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/evalview.egg-info/top_level.txt +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/setup.cfg +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_api.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_celebrations.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_ci_generate_comment.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_diff_engine.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_dogfood_e2e.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_drift_tracker.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_e2e_snapshot_check.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_endpoint_detection.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_evaluator_orchestrator.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_evaluators.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_forbidden_tools.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_generate_cmd.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_golden_store.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_healing.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_init_cmd.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_judge_cache.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_llm_provider.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_main_evaluator.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_mcp_contracts.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_messages.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_model_runtime_detector.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_multi_turn_evaluation.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_pii_evaluator.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_project_state.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_pytest_plugin.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_run_guidance.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_security.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_semantic_diff.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_snapshot_generated_workflow.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_statistical_evaluator.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_telemetry_client.py +0 -0
- {evalview-0.6.0 → evalview-0.7.0}/tests/test_version_check.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: evalview
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.0
|
|
4
4
|
Summary: Open-source testing and regression detection framework for AI agents. Golden baseline diffing, CI/CD integration, works with LangGraph, CrewAI, OpenAI, Anthropic Claude, HuggingFace, Ollama, and MCP.
|
|
5
5
|
Author-email: EvalView Team <hidai@evalview.com>
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -35,6 +35,7 @@ Requires-Dist: httpx>=0.26.0
|
|
|
35
35
|
Requires-Dist: python-dateutil>=2.8.2
|
|
36
36
|
Requires-Dist: python-dotenv>=1.0.0
|
|
37
37
|
Requires-Dist: jinja2>=3.0
|
|
38
|
+
Requires-Dist: jsonschema>=4.0.0
|
|
38
39
|
Provides-Extra: reports
|
|
39
40
|
Requires-Dist: plotly>=5.0; extra == "reports"
|
|
40
41
|
Provides-Extra: watch
|
|
@@ -73,7 +74,7 @@ Dynamic: license-file
|
|
|
73
74
|
<p align="center">
|
|
74
75
|
<img src="assets/logo.png" alt="EvalView" width="350">
|
|
75
76
|
<br>
|
|
76
|
-
<strong>The open-source regression gate for AI agents.</strong><br>
|
|
77
|
+
<strong>The open-source behavior regression gate for AI agents.</strong><br>
|
|
77
78
|
Think Playwright, but for tool-calling and multi-turn AI agents.
|
|
78
79
|
</p>
|
|
79
80
|
|
|
@@ -88,17 +89,16 @@ Dynamic: license-file
|
|
|
88
89
|
|
|
89
90
|
---
|
|
90
91
|
|
|
91
|
-
Your agent can still return `200` and be wrong. A model or provider update can change tool choice, skip a clarification, or degrade output quality without changing your code or breaking a health check. **EvalView catches those silent regressions before users do.**
|
|
92
|
+
Your agent can still return `200` and be wrong. A model or provider update can change tool choice, skip a clarification, or degrade output quality without changing your code or breaking a health check. **EvalView catches those silent regressions before users do — and gives you the loop to investigate them, grade the confidence, and broadcast the verdict to your team.**
|
|
92
93
|
|
|
93
|
-
**
|
|
94
|
+
**You don't need frontier-lab resources to run a serious agent regression loop.** EvalView gives solo devs, startups, and small AI teams the same core discipline: snapshot behavior, detect drift, classify changes, and review or heal them safely.
|
|
94
95
|
|
|
95
|
-
|
|
96
|
+
**Traditional tests tell you if your agent is up. EvalView tells you if it still behaves correctly.** It tracks drift across outputs, tools, model IDs, and runtime fingerprints with graded confidence — not a binary alarm — so you can tell "the provider changed" from "my system regressed."
|
|
97
|
+
|
|
98
|
+
[](https://github.com/user-attachments/assets/96d8b5f7-3561-44a1-86a4-270fb0d1d8a6)
|
|
96
99
|
|
|
97
100
|
**30-second live demo.**
|
|
98
101
|
|
|
99
|
-
If the video does not render in your GitHub app, use the GIF preview below.
|
|
100
|
-
|
|
101
|
-
[](https://github.com/user-attachments/assets/96d8b5f7-3561-44a1-86a4-270fb0d1d8a6)
|
|
102
102
|
|
|
103
103
|
Most eval tools stop at detect and compare. EvalView helps you classify changes, inspect drift, and auto-heal the safe cases.
|
|
104
104
|
|
|
@@ -106,6 +106,13 @@ Most eval tools stop at detect and compare. EvalView helps you classify changes,
|
|
|
106
106
|
- Separate provider/model drift from real system regressions
|
|
107
107
|
- Auto-heal flaky failures with retries, review gates, and audit logs
|
|
108
108
|
|
|
109
|
+
Built for **frontier-lab rigor, startup-team practicality**:
|
|
110
|
+
- targeted behavior runs instead of giant always-on eval suites
|
|
111
|
+
- deterministic diffs first, LLM judgment where it adds signal
|
|
112
|
+
- faster loops from change -> eval -> review -> ship
|
|
113
|
+
|
|
114
|
+
[How we run EvalView with this operating model →](docs/OPERATING_MODEL.md)
|
|
115
|
+
|
|
109
116
|
```
|
|
110
117
|
✓ login-flow PASSED
|
|
111
118
|
⚠ refund-request TOOLS_CHANGED
|
|
@@ -115,6 +122,32 @@ Most eval tools stop at detect and compare. EvalView helps you classify changes,
|
|
|
115
122
|
Score: 85 → 55 Output similarity: 35%
|
|
116
123
|
```
|
|
117
124
|
|
|
125
|
+
The money screen is the one-line **verdict** that lands under every check — a single ship/don't-ship decision derived from the diff, quarantine state, cost delta, and drift confidence:
|
|
126
|
+
|
|
127
|
+
```
|
|
128
|
+
─────────────────────────────────────────────
|
|
129
|
+
VERDICT: 🛑 BLOCK RELEASE
|
|
130
|
+
─────────────────────────────────────────────
|
|
131
|
+
|
|
132
|
+
• 1 regression: billing-dispute
|
|
133
|
+
• 1 test changed behavior: refund-request
|
|
134
|
+
• Cost up 14% vs baseline
|
|
135
|
+
|
|
136
|
+
Likely cause & next actions:
|
|
137
|
+
|
|
138
|
+
1. Rerun statistically to distinguish flake from real drift
|
|
139
|
+
(high severity, high confidence)
|
|
140
|
+
→ evalview check --statistical 5
|
|
141
|
+
|
|
142
|
+
2. Review tool descriptions for: escalate_to_human
|
|
143
|
+
(high severity, high confidence)
|
|
144
|
+
Tool selection changed — usually a prompt edit nudged the model
|
|
145
|
+
→ evalview replay refund-request --trace
|
|
146
|
+
→ evalview golden update refund-request # if the new path is correct
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
Four tiers: `SAFE_TO_SHIP`, `SHIP_WITH_QUARANTINE`, `INVESTIGATE`, `BLOCK_RELEASE`. The verdict is part of `--json` output, the PR comment, and the cloud ship page — CLI, CI, and dashboard all tell the same story.
|
|
150
|
+
|
|
118
151
|
## Quick Start
|
|
119
152
|
|
|
120
153
|
```bash
|
|
@@ -129,6 +162,28 @@ evalview check # Catch regressions after every change
|
|
|
129
162
|
|
|
130
163
|
That's it. Three commands to regression-test any AI agent. `init` auto-detects your agent type (chat, tool-use, multi-step, RAG, coding) and configures the right evaluators, thresholds, and assertions.
|
|
131
164
|
|
|
165
|
+
**After `check`, the investigative loop:**
|
|
166
|
+
|
|
167
|
+
```bash
|
|
168
|
+
evalview since # 2-second brief: what's changed since your last run
|
|
169
|
+
evalview progress --since yesterday # delta report: what improved/regressed
|
|
170
|
+
evalview drift # per-test sparklines + incident markers
|
|
171
|
+
evalview slack-digest --webhook $SLACK # post the daily verdict to your team
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
These four commands turn a red ✗ into an answer — *is it real drift, a known flake, or a provider update?* — before anyone opens Slack. `since` is the habit anchor (daily open-the-terminal glance); `progress` answers "did my changes help?" with a worth-a-commit gate; `drift` visualizes the trend; `slack-digest` broadcasts the verdict. See [Daily Workflow →](#daily-workflow).
|
|
175
|
+
|
|
176
|
+
### Catch silent drift in closed models
|
|
177
|
+
|
|
178
|
+
Worried that `claude-opus-4-5` might behave differently next week without warning? [`evalview model-check`](#model-drift-detection) runs a zero-judge canary suite directly against the provider and tells you exactly when the model has drifted — no agent required, no calibration.
|
|
179
|
+
|
|
180
|
+
```bash
|
|
181
|
+
evalview model-check --model claude-opus-4-5-20251101 # first run saves baseline
|
|
182
|
+
evalview model-check --model claude-opus-4-5-20251101 # next week, detects any change
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
[See the full model-check section →](#model-drift-detection)
|
|
186
|
+
|
|
132
187
|
<details>
|
|
133
188
|
<summary><strong>Other install methods</strong></summary>
|
|
134
189
|
|
|
@@ -168,6 +223,111 @@ evalview init --profile rag # Override auto-detect
|
|
|
168
223
|
|
|
169
224
|
</details>
|
|
170
225
|
|
|
226
|
+
## Daily Workflow
|
|
227
|
+
|
|
228
|
+
Detection is only the first step. EvalView gives you the full investigative loop — so when a test goes red, you can answer *"is it real drift, a known flake, or a provider update?"* in four commands, before anyone opens Slack.
|
|
229
|
+
|
|
230
|
+
**Open the terminal — `evalview since`**
|
|
231
|
+
|
|
232
|
+
```
|
|
233
|
+
╭──────────────────────────── Since your last check ─────────────────────────────╮
|
|
234
|
+
│ │
|
|
235
|
+
│ 95% pass rate across 14 runs │
|
|
236
|
+
│ ⚠ 2 soft change(s) │
|
|
237
|
+
│ ✨ improved: search_cases, summarize_thread │
|
|
238
|
+
│ │
|
|
239
|
+
╰────────────────────────────────────────────────────────────────────────────────╯
|
|
240
|
+
|
|
241
|
+
Drift sparklines (most-declining first)
|
|
242
|
+
▇▆▅▄▃▂▁▂ billing-dispute
|
|
243
|
+
|
|
244
|
+
🔥 Streak: 6 days of clean merges
|
|
245
|
+
|
|
246
|
+
One thing to look at first:
|
|
247
|
+
→ evalview replay billing-dispute
|
|
248
|
+
```
|
|
249
|
+
|
|
250
|
+
`evalview since` is the **2-second habit brief** — one hero number, one concern, one action. It reuses Week 1's fingerprinted history, so the "since your last check" window is accurate whether you ran it 10 minutes ago or 4 days ago. Night owls at 2am and daily shippers at 9am see the same command; the label adapts. It's the command that goes in your `.zshrc` so `evalview since` fires when you open the terminal, before the espresso machine is warm.
|
|
251
|
+
|
|
252
|
+
**Morning — `evalview progress --since yesterday`**
|
|
253
|
+
|
|
254
|
+
```
|
|
255
|
+
✨ 3 test(s) now passing that weren't
|
|
256
|
+
⚠ 1 test(s) regressed
|
|
257
|
+
|
|
258
|
+
Improved:
|
|
259
|
+
+ refund-flow
|
|
260
|
+
+ order-lookup (at a4f2e91)
|
|
261
|
+
|
|
262
|
+
Regressed:
|
|
263
|
+
− billing-dispute
|
|
264
|
+
|
|
265
|
+
Output similarity: 85.20% → 87.50% ↑ +2.30%
|
|
266
|
+
|
|
267
|
+
Worth a commit:
|
|
268
|
+
✓ refund-flow (high confidence)
|
|
269
|
+
→ evalview golden update refund-flow
|
|
270
|
+
```
|
|
271
|
+
|
|
272
|
+
A **"worth a commit" gate** (3+ consecutive passes) keeps you from celebrating flakes at 2am.
|
|
273
|
+
|
|
274
|
+
**Triage — `evalview drift billing-dispute`**
|
|
275
|
+
|
|
276
|
+
```
|
|
277
|
+
Test │ Trend │ Samples │ Slope │ First → Last │ Status
|
|
278
|
+
─────────────────┼──────────────┼─────────┼────────┼──────────────┼────────
|
|
279
|
+
billing-dispute │ ! │ 20 │ -1.5% │ 90% → 80% │ declining
|
|
280
|
+
│ ▇▆▅▄▃▂▁▂▁▂▁ │ │ │ │
|
|
281
|
+
|
|
282
|
+
Most concerning: billing-dispute — slope -1.50% per check over 20 samples
|
|
283
|
+
→ evalview replay billing-dispute --trace
|
|
284
|
+
```
|
|
285
|
+
|
|
286
|
+
Unicode sparklines + OLS slope + **incident markers** (`!`) show *when* the test flipped. Drift is graded `insufficient_history / stable / low / medium / high` — not a binary alarm.
|
|
287
|
+
|
|
288
|
+
**Verdict — `evalview check --statistical 5`**
|
|
289
|
+
|
|
290
|
+
When the verdict layer returns `INVESTIGATE`, a stability-replay recommendation is auto-injected at position #0 of the action list, surviving the severity sort so you never miss it.
|
|
291
|
+
|
|
292
|
+
**Quarantine — `evalview quarantine list`**
|
|
293
|
+
|
|
294
|
+
```
|
|
295
|
+
Test │ Owner │ Age │ Flaky │ Trend │ Status │ Reason
|
|
296
|
+
──────────────────┼─────────┼──────┼───────┼───────┼───────────┼──────────────
|
|
297
|
+
race-condition │ @hidai │ 12d │ 4 │ ↘ │ ⏸ active │ race condition
|
|
298
|
+
db-timeout │ @jane │ 45d │ 8 │ ↗ │ ⏰ STALE │ db timeout
|
|
299
|
+
|
|
300
|
+
⏰ 1 entry stale — review overdue.
|
|
301
|
+
Either fix the underlying flake or remove from quarantine:
|
|
302
|
+
evalview quarantine remove db-timeout
|
|
303
|
+
```
|
|
304
|
+
|
|
305
|
+
Known-flaky tests **don't block CI** — but staleness tracking, owner tags, and a flaky-count trend glyph keep the list honest. Governance built in, not a dumping ground.
|
|
306
|
+
|
|
307
|
+
**Broadcast — `evalview slack-digest --webhook $SLACK_WEBHOOK`**
|
|
308
|
+
|
|
309
|
+
```
|
|
310
|
+
📊 EvalView digest — yesterday
|
|
311
|
+
🟢 95% pass rate across 47 runs
|
|
312
|
+
|
|
313
|
+
Drift
|
|
314
|
+
▇▆▅ billing-dispute
|
|
315
|
+
|
|
316
|
+
⏰ Stale quarantine
|
|
317
|
+
1 overdue
|
|
318
|
+
• db-timeout — @jane — 45d
|
|
319
|
+
|
|
320
|
+
🎯 Next: evalview check --fail-on REGRESSION
|
|
321
|
+
|
|
322
|
+
✓ Digest posted to Slack.
|
|
323
|
+
```
|
|
324
|
+
|
|
325
|
+
Stdlib-only Block Kit post (zero new deps). Fails soft on bad webhooks. Ends with **one actionable command** your team can copy-paste from the channel.
|
|
326
|
+
|
|
327
|
+
---
|
|
328
|
+
|
|
329
|
+
**The loop closes:** detection → investigation → graded verdict → quarantine governance → broadcast. You wake up, run `progress`, triage with `drift`, confirm with `check --statistical`, and the team sees the digest before standup. That's the morning ritual — reach for it before the espresso machine warms up.
|
|
330
|
+
|
|
171
331
|
## Why EvalView?
|
|
172
332
|
|
|
173
333
|
Use LangSmith for observability. Use Braintrust for scoring. **Use EvalView for regression gating.**
|
|
@@ -193,6 +353,9 @@ Use LangSmith for observability. Use Braintrust for scoring. **Use EvalView for
|
|
|
193
353
|
| ⚠️ **TOOLS_CHANGED** | Different tools called | Review the diff |
|
|
194
354
|
| ⚠️ **OUTPUT_CHANGED** | Same tools, output shifted | Review the diff |
|
|
195
355
|
| ❌ **REGRESSION** | Score dropped significantly | Fix before shipping |
|
|
356
|
+
| 📉 **DRIFTING** | Trend sliding with graded confidence (low/med/high) | Run `evalview drift <test>` |
|
|
357
|
+
| 🔎 **INVESTIGATE** | Verdict layer wants statistical replay | Run `evalview check --statistical 5` |
|
|
358
|
+
| ⏳ **QUARANTINED** | Known-flaky, excluded from CI exit code | Fix underlying flake or remove |
|
|
196
359
|
|
|
197
360
|
### Model / Runtime Change Detection
|
|
198
361
|
|
|
@@ -200,7 +363,7 @@ EvalView does more than compare `model_id`.
|
|
|
200
363
|
|
|
201
364
|
- **Declared model change**: adapter-reported model changed from baseline
|
|
202
365
|
- **Runtime fingerprint change**: observed model labels in the trace changed, even when the top-level model name is missing
|
|
203
|
-
- **Coordinated drift**: multiple tests shift together in the same check run, which often points to a silent provider rollout or runtime change
|
|
366
|
+
- **Coordinated drift**: multiple tests shift together in the same check run, which often points to a silent provider rollout or runtime change — now **graded `low / medium / high`** via `DriftTracker.classify_drift`, not a binary alarm
|
|
204
367
|
|
|
205
368
|
When detected, `evalview check` surfaces a run-level signal with a classification (`declared` or `suspected`), confidence level, and evidence from fingerprints, retries, and affected tests.
|
|
206
369
|
|
|
@@ -464,9 +627,13 @@ Works with **LangGraph, CrewAI, OpenAI, Claude, Mistral, HuggingFace, Ollama, MC
|
|
|
464
627
|
|
|
465
628
|
1. **`evalview init`** — detects your running agent, creates a starter test suite
|
|
466
629
|
2. **`evalview snapshot`** — runs tests, saves traces as baselines
|
|
467
|
-
3. **`evalview check`** — replays tests, diffs against baselines, opens HTML report
|
|
468
|
-
4. **`evalview
|
|
469
|
-
5. **`evalview
|
|
630
|
+
3. **`evalview check`** — replays tests, diffs against baselines, emits the ship/don't-ship verdict, opens HTML report
|
|
631
|
+
4. **`evalview since`** — 2-second brief: what's changed since your last run (the daily habit anchor)
|
|
632
|
+
5. **`evalview watch`** — re-runs checks on every file save
|
|
633
|
+
6. **`evalview monitor`** — continuous checks in production with Slack alerts
|
|
634
|
+
7. **`evalview progress --since`** — diff any two points in history with a "worth a commit" gate
|
|
635
|
+
8. **`evalview drift`** — per-test sparklines, OLS slope, and incident markers
|
|
636
|
+
9. **`evalview slack-digest`** — post the daily verdict to your team channel
|
|
470
637
|
|
|
471
638
|
<details>
|
|
472
639
|
<summary><strong>Snapshot management</strong></summary>
|
|
@@ -491,16 +658,141 @@ evalview monitor # Check every 5 min
|
|
|
491
658
|
evalview monitor --dashboard # Live terminal dashboard
|
|
492
659
|
evalview monitor --slack-webhook https://hooks.slack.com/services/...
|
|
493
660
|
evalview monitor --history monitor.jsonl # JSONL for dashboards
|
|
661
|
+
evalview monitor --incidents # Log confirmed regressions for `evalview autopr`
|
|
494
662
|
```
|
|
495
663
|
|
|
496
664
|
New regressions trigger Slack alerts. Recoveries send all-clear. No spam on persistent failures.
|
|
497
665
|
|
|
666
|
+
**Every alert is a promise.** The monitor requires **two consecutive failing cycles** before it pages a human — a single blip self-resolves silently and never interrupts anyone. If a test must alert on the first failure (auth, payments, PII, refund paths), mark it `gate: strict` in its YAML and it bypasses the gate, re-alerting every cycle until it passes.
|
|
667
|
+
|
|
668
|
+
Suppressed failures are never hidden: `evalview slack-digest` renders a Noise section listing every test the gate swallowed, how many times it self-resolved, and a visible false-positive rate (`3 suppressed / 12 fired = 25% noise`). See [`evalview/core/noise_tracker.py`](evalview/core/noise_tracker.py) for the full design — confirmation gate, coordinated-incident collapse, and the `.evalview/noise.jsonl` metric.
|
|
669
|
+
|
|
498
670
|
[Monitor config options →](docs/CLI_REFERENCE.md)
|
|
499
671
|
|
|
672
|
+
## Auto-PR Regression Tests From Production Incidents
|
|
673
|
+
|
|
674
|
+
`evalview autopr` closes the loop: **production failure → pinned regression test → pull request**, with zero LLM calls and zero manual YAML writing.
|
|
675
|
+
|
|
676
|
+
```bash
|
|
677
|
+
evalview monitor --incidents # Monitor writes .evalview/incidents.jsonl
|
|
678
|
+
evalview autopr --dry-run # Preview what would be generated
|
|
679
|
+
evalview autopr # Write tests/regressions/*.yaml
|
|
680
|
+
evalview autopr --open-pr # + commit + push + gh pr create
|
|
681
|
+
```
|
|
682
|
+
|
|
683
|
+
The synthesizer is **pure and deterministic** — no network, no LLM — so it runs instantly in CI. For each confirmed regression in `.evalview/incidents.jsonl` it:
|
|
684
|
+
|
|
685
|
+
- builds a `tests/regressions/<slug>.yaml` that pins the query, the baseline tool sequence (`expected.tools`), the newly-appeared tools (`forbidden_tools`), and short phrases from the bad output (`output.not_contains`)
|
|
686
|
+
- tags the test `suite_type: regression`, `gate: strict`, and stamps the incident metadata into `meta.incident` so `autopr` can skip it on subsequent runs
|
|
687
|
+
- defaults to `min_score: 90` — regression tests are a safety net, not a capability benchmark
|
|
688
|
+
|
|
689
|
+
```yaml
|
|
690
|
+
# tests/regressions/refund-request-b3c4d5e6.yaml (auto-generated)
|
|
691
|
+
name: regression_refund-request_2026-04-14
|
|
692
|
+
description: Auto-generated from production incident (REGRESSION) at 2026-04-14T12:34:56Z ...
|
|
693
|
+
input:
|
|
694
|
+
query: "I want a refund for order #123"
|
|
695
|
+
expected:
|
|
696
|
+
tools: [lookup_order, check_policy, process_refund]
|
|
697
|
+
forbidden_tools: [escalate_to_human] # appeared only in the failing trace
|
|
698
|
+
output:
|
|
699
|
+
not_contains: ["Sure, I've processed your refund for $999."]
|
|
700
|
+
thresholds:
|
|
701
|
+
min_score: 90.0
|
|
702
|
+
suite_type: regression
|
|
703
|
+
gate: strict
|
|
704
|
+
tags: [incident, autopr]
|
|
705
|
+
```
|
|
706
|
+
|
|
707
|
+
**Wire it into GitHub Actions:** copy [`examples/github-workflow-autopr.yml`](examples/github-workflow-autopr.yml) to `.github/workflows/evalview-autopr.yml` — the workflow runs `monitor` then `autopr --open-pr --require-new` on a schedule. Every production regression arrives as a reviewable PR, and your hallucination test suite grows by itself.
|
|
708
|
+
|
|
709
|
+
**The CLI is fully local.** `evalview monitor` + `evalview autopr` run entirely on your machine — local files, `gh pr create`, no network calls, no cloud account required. The primitive is free and open-source forever.
|
|
710
|
+
|
|
711
|
+
## Model Drift Detection
|
|
712
|
+
|
|
713
|
+
Closed models update silently. `evalview model-check` is a dedicated command that runs a fixed structural canary suite directly against the provider and tells you when the model itself has changed — no agent, no LLM judge, no calibration required.
|
|
714
|
+
|
|
715
|
+
```bash
|
|
716
|
+
# Save a baseline snapshot the day you deploy
|
|
717
|
+
evalview model-check --model claude-opus-4-5-20251101
|
|
718
|
+
|
|
719
|
+
# Run it weekly — detects any behavioral change against that baseline
|
|
720
|
+
evalview model-check --model claude-opus-4-5-20251101
|
|
721
|
+
```
|
|
722
|
+
|
|
723
|
+
**Example output when drift is detected:**
|
|
724
|
+
|
|
725
|
+
```
|
|
726
|
+
EvalView model-check
|
|
727
|
+
Model: claude-opus-4-5-20251101
|
|
728
|
+
Provider: anthropic
|
|
729
|
+
Suite: canary v1.public (15 prompts, sha256:6b8e925a5543…)
|
|
730
|
+
Runs/prompt: 1
|
|
731
|
+
Temperature: 0.0
|
|
732
|
+
Cost: $0.0228
|
|
733
|
+
|
|
734
|
+
vs reference (2026-04-10, 7d ago)
|
|
735
|
+
Drift: MODEL (MEDIUM confidence)
|
|
736
|
+
Pass rate: 15/15 → 13/15 (-13.3%)
|
|
737
|
+
Flipped: tool_choice_refund_first_step, json_schema_order_summary
|
|
738
|
+
|
|
739
|
+
vs previous (2026-04-17, 0d ago)
|
|
740
|
+
Drift: NONE
|
|
741
|
+
Pass rate: 13/15 → 13/15 (+0.0%)
|
|
742
|
+
```
|
|
743
|
+
|
|
744
|
+
**How it works:**
|
|
745
|
+
|
|
746
|
+
| Check type | What it catches |
|
|
747
|
+
|------------|-----------------|
|
|
748
|
+
| **Tool choice** | Did the model pick the right tool, in the right order? |
|
|
749
|
+
| **JSON schema** | Does the output still match the expected structure? |
|
|
750
|
+
| **Refusal** | Did the model refuse when it should (or comply when it should)? |
|
|
751
|
+
| **Exact match** | Does the response match a regex anchor? |
|
|
752
|
+
|
|
753
|
+
Every check runs at `temperature=0` for determinism. Drift is classified as **NONE / WEAK / MEDIUM / STRONG** based on how many prompts flipped pass↔fail. No judge — the signal is structural, not probabilistic.
|
|
754
|
+
|
|
755
|
+
```bash
|
|
756
|
+
evalview model-check --model claude-opus-4-5-20251101 --dry-run # Cost estimate before running
|
|
757
|
+
evalview model-check --model claude-opus-4-5-20251101 --pin # Pin this run as the new reference
|
|
758
|
+
evalview model-check --model claude-opus-4-5-20251101 --reset-reference # Start a fresh baseline
|
|
759
|
+
evalview model-check --model claude-opus-4-5-20251101 --json # Machine-readable output for CI
|
|
760
|
+
evalview model-check --model claude-opus-4-5-20251101 --suite my-canary.yaml # Bring your own suite
|
|
761
|
+
```
|
|
762
|
+
|
|
763
|
+
Ships with a **bundled 15-prompt public canary** covering tool selection, JSON schema, refusal behavior, and exact match. Add your own prompts with `--suite`. v1 supports **Anthropic**; OpenAI/Mistral/Cohere land in v1.1.
|
|
764
|
+
|
|
765
|
+
[Full reference → docs/MODEL_CHECK.md](docs/MODEL_CHECK.md)
|
|
766
|
+
|
|
767
|
+
## Simulation + Decision Rationale
|
|
768
|
+
|
|
769
|
+
Two features that close the gaps the April 2026 agent-eval reports kept calling out: pre-flight what-if testing and structured "why did the agent branch?" logging.
|
|
770
|
+
|
|
771
|
+
**Pre-flight simulation** — `evalview simulate` runs your test suite hermetically against declared mocks (tool calls, LLM responses, HTTP). Deterministic seed, zero cost, works in CI. Declare mocks in the test YAML, fan out with `--variants N`:
|
|
772
|
+
|
|
773
|
+
```bash
|
|
774
|
+
evalview simulate tests/ --variants 5 --seed 42
|
|
775
|
+
```
|
|
776
|
+
|
|
777
|
+
**Decision rationale** — every tool_choice / branch gets recorded with the chosen option, alternatives considered, and any model-reported reasoning (Anthropic `thinking` blocks auto-captured). Grouped across runs by `input_hash` so cloud analytics surfaces decision drift before your users notice. Local HTML replay shows it inline.
|
|
778
|
+
|
|
779
|
+
Supported adapters: Anthropic, OpenAI Assistants, LangGraph, CrewAI native.
|
|
780
|
+
|
|
781
|
+
[`docs/SIMULATE.md`](docs/SIMULATE.md) · [`docs/RATIONALE.md`](docs/RATIONALE.md) · [`examples/simulation/`](examples/simulation)
|
|
782
|
+
|
|
500
783
|
## Key Features
|
|
501
784
|
|
|
502
785
|
| Feature | Description | Docs |
|
|
503
786
|
|---------|-------------|------|
|
|
787
|
+
| **Progress command** | `evalview progress --since <date\|sha>` — improved/regressed with "worth a commit" gate | [Above](#daily-workflow) |
|
|
788
|
+
| **Drift command** | `evalview drift` — unicode sparklines, OLS slope, incident markers | [Above](#daily-workflow) |
|
|
789
|
+
| **Slack digest** | `evalview slack-digest` — stdlib Block Kit post with one actionable next-step | [Above](#daily-workflow) |
|
|
790
|
+
| **Flake quarantine** | Known-flaky tests don't block CI; staleness tracking, owner tags, governance | [Above](#daily-workflow) |
|
|
791
|
+
| **Release verdict layer** | Graded drift confidence + auto-injected stability recommendation | [Above](#daily-workflow) |
|
|
792
|
+
| **Recommendation engine** | Suggests the next command from verdict, drift class, and history | [Above](#daily-workflow) |
|
|
793
|
+
| **Model drift detection** | `model-check` — zero-judge canary suite that catches silent model updates | [Docs](#model-drift-detection) |
|
|
794
|
+
| **Simulation harness** | `evalview simulate` — hermetic what-if runs against declared mocks, with `--variants N` fan-out | [Docs](docs/SIMULATE.md) |
|
|
795
|
+
| **Decision rationale** | Structured `tool_choice` / `branch` logging with cross-run grouping for decision-drift detection | [Docs](docs/RATIONALE.md) |
|
|
504
796
|
| **Assertion wizard** | Analyze captured traffic, suggest smart assertions automatically | [Above](#assertion-wizard--tests-from-real-traffic) |
|
|
505
797
|
| **Auto-variant discovery** | Run N times, cluster paths, save valid variants | [Above](#auto-variant-discovery--solve-non-determinism) |
|
|
506
798
|
| **Auto-heal** | Retry flakes, propose variants, escalate structural changes | [Above](#auto-heal--fix-flakes-without-leaving-ci) |
|
|
@@ -511,6 +803,7 @@ New regressions trigger Slack alerts. Recoveries send all-clear. No spam on pers
|
|
|
511
803
|
| **Multi-reference baselines** | Up to 5 variants for non-deterministic agents | [Docs](docs/GOLDEN_TRACES.md) |
|
|
512
804
|
| **`forbidden_tools`** | Safety contracts — hard-fail on any violation | [Docs](docs/YAML_SCHEMA.md) |
|
|
513
805
|
| **Watch mode** | `evalview watch` — re-run checks on file save, with dashboard | [Docs](#watch-mode) |
|
|
806
|
+
| **Model comparison** | `run_eval` / `compare_models` — test one query across N models in parallel | [Docs](#model-comparison) |
|
|
514
807
|
| **Python API** | `gate()` / `gate_async()` — programmatic regression checks | [Docs](#python-api) |
|
|
515
808
|
| **PR comments + alerts** | Cost/latency spikes, model changes, collapsible diffs | [Docs](docs/CI_CD.md) |
|
|
516
809
|
| **Terminal dashboard** | Scorecard, sparkline trends, confidence scoring | — |
|
|
@@ -523,6 +816,7 @@ New regressions trigger Slack alerts. Recoveries send all-clear. No spam on pers
|
|
|
523
816
|
| **Multi-turn capture** | `capture --multi-turn` records conversations as tests | [Docs](#multi-turn-testing) |
|
|
524
817
|
| **Semantic similarity** | Embedding-based output comparison | [Docs](docs/EVALUATION_METRICS.md) |
|
|
525
818
|
| **Production monitoring** | `evalview monitor --dashboard` with Slack alerts and JSONL history | [Docs](#production-monitoring) |
|
|
819
|
+
| **Auto-PR regression tests** | `evalview autopr` turns `.evalview/incidents.jsonl` into pinned regression tests + PRs | [Docs](#auto-pr-regression-tests-from-production-incidents) |
|
|
526
820
|
| **A/B comparison** | `evalview compare --v1 <url> --v2 <url>` | [Docs](docs/CLI_REFERENCE.md) |
|
|
527
821
|
| **Test generation** | `evalview generate` — discovers your agent's domain, generates relevant tests | [Docs](docs/TEST_GENERATION.md) |
|
|
528
822
|
| **Per-turn judge scoring** | Multi-turn output quality scored per turn with conversation context | [Docs](#multi-turn-testing) |
|
|
@@ -533,6 +827,7 @@ New regressions trigger Slack alerts. Recoveries send all-clear. No spam on pers
|
|
|
533
827
|
| **Verified cost tracking** | Per-test cost breakdown with model pricing rates | [Docs](docs/COST_TRACKING.md) |
|
|
534
828
|
| **Judge model picker** | Choose GPT, Claude, Gemini, DeepSeek, or Ollama (free) | [Docs](docs/EVALUATION_METRICS.md) |
|
|
535
829
|
| **Pytest plugin** | `evalview_check` fixture for standard pytest | [Docs](#pytest-plugin) |
|
|
830
|
+
| **Model comparison** | `run_eval` / `compare_models` — parametrize tests across models, auto-detect provider | [Docs](#model-comparison) |
|
|
536
831
|
| **GitHub Actions job summary** | Results visible in Actions UI, not just PR comments | [Docs](docs/CI_CD.md) |
|
|
537
832
|
| **Git hooks** | Pre-push regression blocking, zero CI config | [Docs](docs/CI_CD.md) |
|
|
538
833
|
| **LLM judge caching** | ~80% cost reduction in statistical mode | [Docs](docs/EVALUATION_METRICS.md) |
|
|
@@ -621,6 +916,71 @@ pip install evalview # Plugin registers automatically
|
|
|
621
916
|
pytest # Runs alongside your existing tests
|
|
622
917
|
```
|
|
623
918
|
|
|
919
|
+
## Model Comparison
|
|
920
|
+
|
|
921
|
+
Test the same task across multiple models with one parametrized test. No config files — just a model name and a query.
|
|
922
|
+
|
|
923
|
+
```python
|
|
924
|
+
import pytest
|
|
925
|
+
import evalview
|
|
926
|
+
|
|
927
|
+
@pytest.mark.parametrize("model", ["claude-opus-4-6", "gpt-4o", "claude-sonnet-4-6"])
|
|
928
|
+
def test_my_task(model):
|
|
929
|
+
result = evalview.run_eval(model, query="Summarize this contract in one sentence.")
|
|
930
|
+
assert evalview.score(result) > 0.8
|
|
931
|
+
```
|
|
932
|
+
|
|
933
|
+
Provider is auto-detected from the model name. Requires `ANTHROPIC_API_KEY` / `OPENAI_API_KEY` depending on which models you use.
|
|
934
|
+
|
|
935
|
+
**Score against expected output** — token-overlap similarity, no LLM judge needed:
|
|
936
|
+
|
|
937
|
+
```python
|
|
938
|
+
result = evalview.run_eval(
|
|
939
|
+
"gpt-4o",
|
|
940
|
+
query="What language is Python?",
|
|
941
|
+
expected="Python is a high-level interpreted language.",
|
|
942
|
+
threshold=0.4,
|
|
943
|
+
)
|
|
944
|
+
```
|
|
945
|
+
|
|
946
|
+
**Custom scorer** — assert specific behavior:
|
|
947
|
+
|
|
948
|
+
```python
|
|
949
|
+
def has_json(output, expected):
|
|
950
|
+
import json, re
|
|
951
|
+
m = re.search(r"\{.*?\}", output, re.DOTALL)
|
|
952
|
+
try: return 1.0 if json.loads(m.group()) else 0.0
|
|
953
|
+
except: return 0.0
|
|
954
|
+
|
|
955
|
+
result = evalview.run_eval("claude-opus-4-6", query="Return JSON: {name, age}", scorer=has_json)
|
|
956
|
+
assert evalview.score(result) == 1.0
|
|
957
|
+
```
|
|
958
|
+
|
|
959
|
+
**Run all models in parallel and compare:**
|
|
960
|
+
|
|
961
|
+
```python
|
|
962
|
+
results = evalview.compare_models(
|
|
963
|
+
query="Explain quantum entanglement in one sentence.",
|
|
964
|
+
models=["claude-opus-4-6", "gpt-4o", "claude-sonnet-4-6"],
|
|
965
|
+
)
|
|
966
|
+
evalview.print_comparison_table(results) # Rich table: score, latency, cost
|
|
967
|
+
best = results[0] # sorted best-first
|
|
968
|
+
```
|
|
969
|
+
|
|
970
|
+
```
|
|
971
|
+
┏━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━┓
|
|
972
|
+
┃ Model ┃ Score ┃ Latency ┃ Cost ┃ Pass? ┃
|
|
973
|
+
┡━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━┩
|
|
974
|
+
│ claude-opus-4-6 │ 1.00 │ 842ms │ $0.00312 │ ✓ │
|
|
975
|
+
│ gpt-4o │ 1.00 │ 631ms │ $0.00087 │ ✓ │
|
|
976
|
+
│ claude-sonnet-4-6 │ 1.00 │ 514ms │ $0.00063 │ ✓ │
|
|
977
|
+
└────────────────────┴───────┴──────────┴───────────┴───────┘
|
|
978
|
+
```
|
|
979
|
+
|
|
980
|
+
`ModelResult` fields: `.model`, `.output`, `.score`, `.latency_ms`, `.cost_usd`, `.passed`, `.error`
|
|
981
|
+
|
|
982
|
+
[Full example →](examples/model_comparison_test.py)
|
|
983
|
+
|
|
624
984
|
## Claude Code (MCP)
|
|
625
985
|
|
|
626
986
|
```bash
|
|
@@ -665,6 +1025,7 @@ Then just ask Claude: "did my refactor break anything?" and it runs `run_check`
|
|
|
665
1025
|
|---|---|---|
|
|
666
1026
|
| [Getting Started](docs/GETTING_STARTED.md) | [Golden Traces](docs/GOLDEN_TRACES.md) | [CI/CD](docs/CI_CD.md) |
|
|
667
1027
|
| [CLI Reference](docs/CLI_REFERENCE.md) | [Evaluation Metrics](docs/EVALUATION_METRICS.md) | [MCP Contracts](docs/MCP_CONTRACTS.md) |
|
|
1028
|
+
| [Simulation](docs/SIMULATE.md) | [Decision Rationale](docs/RATIONALE.md) | |
|
|
668
1029
|
| [Agent Instructions](AGENT_INSTRUCTIONS.md) | [Agent Recipes](docs/agent-recipes/README.md) | [Ollama Recipe](docs/agent-recipes/integrate-ollama.md) |
|
|
669
1030
|
| [FAQ](docs/FAQ.md) | [Test Generation](docs/TEST_GENERATION.md) | [Skills Testing](docs/SKILLS_TESTING.md) |
|
|
670
1031
|
| [YAML Schema](docs/YAML_SCHEMA.md) | [Statistical Mode](docs/STATISTICAL_MODE.md) | [Chat Mode](docs/CHAT_MODE.md) |
|