evalview 0.6.1__tar.gz → 0.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {evalview-0.6.1/evalview.egg-info → evalview-0.7.0}/PKG-INFO +371 -7
- {evalview-0.6.1 → evalview-0.7.0}/README.md +369 -6
- evalview-0.7.0/evalview/__init__.py +15 -0
- evalview-0.7.0/evalview/adapters/aider_adapter.py +535 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/adapters/anthropic_adapter.py +33 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/adapters/crewai_native_adapter.py +34 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/adapters/langgraph_adapter.py +47 -2
- {evalview-0.6.1 → evalview-0.7.0}/evalview/adapters/openai_assistants_adapter.py +18 -0
- evalview-0.7.0/evalview/adapters/opencode_adapter.py +387 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/adapters/registry.py +14 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/api.py +13 -1
- evalview-0.7.0/evalview/benchmarks/canary/README.md +93 -0
- evalview-0.7.0/evalview/benchmarks/canary/__init__.py +18 -0
- evalview-0.7.0/evalview/benchmarks/canary/suite.v1.held-out.yaml +66 -0
- evalview-0.7.0/evalview/benchmarks/canary/suite.v1.public.yaml +214 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/chat.py +23 -7
- {evalview-0.6.1 → evalview-0.7.0}/evalview/ci/comment.py +255 -2
- {evalview-0.6.1 → evalview-0.7.0}/evalview/cli.py +37 -1
- evalview-0.7.0/evalview/cloud/push.py +328 -0
- evalview-0.7.0/evalview/commands/autopr_cmd.py +526 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/check_cmd.py +528 -17
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/check_display.py +318 -18
- evalview-0.7.0/evalview/commands/drift_cmd.py +335 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/listing_cmd.py +4 -6
- evalview-0.7.0/evalview/commands/log_cmd.py +212 -0
- evalview-0.7.0/evalview/commands/model_check_cmd.py +691 -0
- evalview-0.7.0/evalview/commands/model_check_render.py +149 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/monitor_cmd.py +406 -115
- evalview-0.7.0/evalview/commands/progress_cmd.py +413 -0
- evalview-0.7.0/evalview/commands/quarantine_cmd.py +191 -0
- evalview-0.7.0/evalview/commands/replay_trace_cmd.py +257 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/run/_adapters.py +40 -2
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/run/_cmd.py +49 -7
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/run/_reporter.py +63 -5
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/shared.py +119 -126
- evalview-0.7.0/evalview/commands/simulate_cmd.py +213 -0
- evalview-0.7.0/evalview/commands/since_cmd.py +536 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/skill_cmd.py +20 -2
- evalview-0.7.0/evalview/commands/slack_digest_cmd.py +423 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/snapshot_cmd.py +133 -59
- evalview-0.7.0/evalview/compare.py +486 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/core/adapter_factory.py +17 -1
- evalview-0.7.0/evalview/core/behavioral_anomalies.py +478 -0
- evalview-0.7.0/evalview/core/benchmark_hardening.py +550 -0
- evalview-0.7.0/evalview/core/canary_suite.py +171 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/core/celebrations.py +3 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/core/config.py +17 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/core/diff.py +8 -0
- evalview-0.7.0/evalview/core/discord_notifier.py +139 -0
- evalview-0.7.0/evalview/core/drift_classifier.py +205 -0
- evalview-0.7.0/evalview/core/drift_kind.py +49 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/core/drift_tracker.py +168 -1
- evalview-0.7.0/evalview/core/model_check_scoring.py +389 -0
- evalview-0.7.0/evalview/core/model_provider_runner.py +224 -0
- evalview-0.7.0/evalview/core/model_snapshots.py +321 -0
- evalview-0.7.0/evalview/core/noise_tracker.py +529 -0
- evalview-0.7.0/evalview/core/observability.py +233 -0
- evalview-0.7.0/evalview/core/quarantine.py +451 -0
- evalview-0.7.0/evalview/core/rationale.py +226 -0
- evalview-0.7.0/evalview/core/recommendations.py +337 -0
- evalview-0.7.0/evalview/core/regression_synth.py +271 -0
- evalview-0.7.0/evalview/core/replay_pipeline.py +330 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/core/root_cause.py +243 -8
- evalview-0.7.0/evalview/core/simulation.py +337 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/core/slack_notifier.py +43 -5
- evalview-0.7.0/evalview/core/turn_coherence.py +567 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/core/types.py +253 -1
- evalview-0.7.0/evalview/core/verdict.py +242 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/evaluators/cost_evaluator.py +7 -2
- {evalview-0.6.1 → evalview-0.7.0}/evalview/evaluators/evaluator.py +35 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/mcp_server.py +8 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/pytest_plugin.py +38 -1
- {evalview-0.6.1 → evalview-0.7.0}/evalview/reporters/console_reporter.py +5 -1
- {evalview-0.6.1 → evalview-0.7.0}/evalview/reporters/html_reporter.py +293 -10
- {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/types.py +15 -1
- evalview-0.7.0/evalview/templates/patterns/code-generation.yaml +31 -0
- evalview-0.7.0/evalview/templates/patterns/cost-budget.yaml +20 -0
- evalview-0.7.0/evalview/templates/patterns/customer-support.yaml +31 -0
- evalview-0.7.0/evalview/templates/patterns/data-analysis.yaml +31 -0
- evalview-0.7.0/evalview/templates/patterns/latency-budget.yaml +20 -0
- evalview-0.7.0/evalview/templates/patterns/multi-turn-memory.yaml +28 -0
- evalview-0.7.0/evalview/templates/patterns/output-format.yaml +23 -0
- evalview-0.7.0/evalview/templates/patterns/rag-citation.yaml +29 -0
- evalview-0.7.0/evalview/templates/patterns/rag-grounding.yaml +27 -0
- evalview-0.7.0/evalview/templates/patterns/research-synthesis.yaml +34 -0
- evalview-0.7.0/evalview/templates/patterns/safety-refusal.yaml +28 -0
- evalview-0.7.0/evalview/templates/patterns/tool-error-handling.yaml +26 -0
- evalview-0.7.0/evalview/templates/patterns/tool-not-called.yaml +22 -0
- evalview-0.7.0/evalview/templates/patterns/tool-sequence.yaml +26 -0
- evalview-0.7.0/evalview/templates/patterns/wrong-tool-chosen.yaml +23 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/test_generation.py +5 -1
- {evalview-0.6.1 → evalview-0.7.0}/evalview/visualization/generators.py +314 -10
- {evalview-0.6.1 → evalview-0.7.0/evalview.egg-info}/PKG-INFO +371 -7
- {evalview-0.6.1 → evalview-0.7.0}/evalview.egg-info/SOURCES.txt +75 -1
- {evalview-0.6.1 → evalview-0.7.0}/evalview.egg-info/requires.txt +1 -0
- {evalview-0.6.1 → evalview-0.7.0}/pyproject.toml +9 -1
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_adapters.py +80 -0
- evalview-0.7.0/tests/test_autopr.py +475 -0
- evalview-0.7.0/tests/test_behavioral_anomalies.py +457 -0
- evalview-0.7.0/tests/test_benchmark_hardening.py +484 -0
- evalview-0.7.0/tests/test_canary_suite.py +171 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_check_cmd.py +224 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_check_pipeline.py +81 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_config_models.py +1 -0
- evalview-0.7.0/tests/test_drift_classifier.py +148 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_evaluator_accuracy.py +2 -2
- evalview-0.7.0/tests/test_model_check_cmd.py +796 -0
- evalview-0.7.0/tests/test_model_check_scoring.py +279 -0
- evalview-0.7.0/tests/test_model_snapshots.py +272 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_monitor.py +176 -1
- evalview-0.7.0/tests/test_noise_tracker.py +449 -0
- evalview-0.7.0/tests/test_observability_regressions.py +330 -0
- evalview-0.7.0/tests/test_rationale.py +317 -0
- evalview-0.7.0/tests/test_replay_pipeline.py +327 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_root_cause.py +381 -22
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_run_cmd_tags.py +2 -2
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_runner.py +8 -2
- evalview-0.7.0/tests/test_simulation.py +359 -0
- evalview-0.7.0/tests/test_snapshot_json_output.py +247 -0
- evalview-0.7.0/tests/test_turn_coherence.py +409 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_types.py +261 -0
- evalview-0.7.0/tests/test_verdict.py +228 -0
- evalview-0.7.0/tests/test_week1_integration.py +330 -0
- evalview-0.7.0/tests/test_week2_quarantine.py +222 -0
- evalview-0.7.0/tests/test_week2_review_fixes.py +284 -0
- evalview-0.7.0/tests/test_week2_since_and_comment.py +344 -0
- evalview-0.7.0/tests/test_week3_progress_and_drift.py +610 -0
- evalview-0.6.1/evalview/__init__.py +0 -6
- {evalview-0.6.1 → evalview-0.7.0}/LICENSE +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/NOTICE +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/__main__.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/adapters/__init__.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/adapters/base.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/adapters/cohere_adapter.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/adapters/crewai_adapter.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/adapters/goose_adapter.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/adapters/http_adapter.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/adapters/huggingface_adapter.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/adapters/mcp_adapter.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/adapters/mistral_adapter.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/adapters/ollama_adapter.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/adapters/openclaw_adapter.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/adapters/pydantic_ai_adapter.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/adapters/tapescope_adapter.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/benchmarks/__init__.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/chat_runtime.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/chat_slash.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/ci/__init__.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/cloud/__init__.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/cloud/auth.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/cloud/client.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/__init__.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/add_cmd.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/assertion_wizard.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/badge_cmd.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/baseline_cmd.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/benchmark_cmd.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/capture_cmd.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/chat_cmd.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/ci_cmd.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/cloud_cmd.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/demo_cmd.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/expand_cmd.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/feedback_cmd.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/generate_cmd.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/golden_cmd.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/gym_cmd.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/hooks_cmd.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/import_cmd.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/init_cmd.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/judge_cmd.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/mcp_cmd.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/openclaw_cmd.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/run/__init__.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/run/_executor.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/run/_runner.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/skill_agent_runner.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/telemetry_cmd.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/traces_cmd.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/trends_cmd.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/visual_cmd.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/watch_cmd.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/core/__init__.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/core/budget.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/core/contract_diff.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/core/dashboard.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/core/eval_profiles.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/core/golden.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/core/healing.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/core/judge_cache.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/core/llm_configs.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/core/llm_provider.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/core/loader.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/core/mcp_contract.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/core/messages.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/core/model_runtime_detector.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/core/parallel.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/core/pricing.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/core/project_state.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/core/retry.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/core/runner.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/core/security.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/core/semantic_diff.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/core/test_quality.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/core/tool_categories.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/core/tracing.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/core/variant_clusterer.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/core/watcher.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/evaluators/__init__.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/evaluators/hallucination_evaluator.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/evaluators/latency_evaluator.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/evaluators/output_evaluator.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/evaluators/pii_evaluator.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/evaluators/safety_evaluator.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/evaluators/sequence_evaluator.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/evaluators/statistical_evaluator.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/evaluators/tool_call_evaluator.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/expander.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/exporters/__init__.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/exporters/html_exporter.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/importers/__init__.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/importers/log_importer.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/openclaw.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/recorder.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/reporters/__init__.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/reporters/json_reporter.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/reporters/trace_live_reporter.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/reporters/trace_reporter.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/__init__.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/adapters/__init__.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/adapters/base.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/adapters/claude_code_adapter.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/adapters/codex_adapter.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/adapters/crewai_adapter.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/adapters/custom_adapter.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/adapters/langgraph_adapter.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/adapters/openai_assistants_adapter.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/adapters/openclaw_adapter.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/adapters/registry.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/agent_runner.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/agent_types.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/constants.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/evaluators/__init__.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/evaluators/deterministic.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/evaluators/orchestrator.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/evaluators/rubric.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/parser.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/runner.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/security_scanner.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/test_generator.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/test_helpers.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/ui_utils.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/validator.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/storage/__init__.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/storage/database.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/telemetry/__init__.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/telemetry/client.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/telemetry/config.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/telemetry/decorators.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/telemetry/events.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/trace_cmd/__init__.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/trace_cmd/collector.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/trace_cmd/patcher.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/trace_cmd/runner.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/tracking/__init__.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/tracking/database.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/tracking/regression.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/version_check.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview/visualization/__init__.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview.egg-info/dependency_links.txt +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview.egg-info/entry_points.txt +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/evalview.egg-info/top_level.txt +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/setup.cfg +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_api.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_celebrations.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_ci_generate_comment.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_diff_engine.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_dogfood_e2e.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_drift_tracker.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_e2e_snapshot_check.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_endpoint_detection.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_evaluator_orchestrator.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_evaluators.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_forbidden_tools.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_generate_cmd.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_golden_store.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_healing.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_init_cmd.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_judge_cache.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_llm_provider.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_loader.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_main_evaluator.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_mcp_contracts.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_mcp_server.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_messages.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_model_runtime_detector.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_multi_turn_evaluation.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_new_features.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_pii_evaluator.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_project_state.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_pytest_plugin.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_run_guidance.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_security.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_semantic_diff.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_snapshot_generated_workflow.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_statistical_evaluator.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_telemetry_client.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_version_check.py +0 -0
- {evalview-0.6.1 → evalview-0.7.0}/tests/test_visualization_generators.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: evalview
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.0
|
|
4
4
|
Summary: Open-source testing and regression detection framework for AI agents. Golden baseline diffing, CI/CD integration, works with LangGraph, CrewAI, OpenAI, Anthropic Claude, HuggingFace, Ollama, and MCP.
|
|
5
5
|
Author-email: EvalView Team <hidai@evalview.com>
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -35,6 +35,7 @@ Requires-Dist: httpx>=0.26.0
|
|
|
35
35
|
Requires-Dist: python-dateutil>=2.8.2
|
|
36
36
|
Requires-Dist: python-dotenv>=1.0.0
|
|
37
37
|
Requires-Dist: jinja2>=3.0
|
|
38
|
+
Requires-Dist: jsonschema>=4.0.0
|
|
38
39
|
Provides-Extra: reports
|
|
39
40
|
Requires-Dist: plotly>=5.0; extra == "reports"
|
|
40
41
|
Provides-Extra: watch
|
|
@@ -88,9 +89,11 @@ Dynamic: license-file
|
|
|
88
89
|
|
|
89
90
|
---
|
|
90
91
|
|
|
91
|
-
Your agent can still return `200` and be wrong. A model or provider update can change tool choice, skip a clarification, or degrade output quality without changing your code or breaking a health check. **EvalView catches those silent regressions before users do.**
|
|
92
|
+
Your agent can still return `200` and be wrong. A model or provider update can change tool choice, skip a clarification, or degrade output quality without changing your code or breaking a health check. **EvalView catches those silent regressions before users do — and gives you the loop to investigate them, grade the confidence, and broadcast the verdict to your team.**
|
|
92
93
|
|
|
93
|
-
**
|
|
94
|
+
**You don't need frontier-lab resources to run a serious agent regression loop.** EvalView gives solo devs, startups, and small AI teams the same core discipline: snapshot behavior, detect drift, classify changes, and review or heal them safely.
|
|
95
|
+
|
|
96
|
+
**Traditional tests tell you if your agent is up. EvalView tells you if it still behaves correctly.** It tracks drift across outputs, tools, model IDs, and runtime fingerprints with graded confidence — not a binary alarm — so you can tell "the provider changed" from "my system regressed."
|
|
94
97
|
|
|
95
98
|
[](https://github.com/user-attachments/assets/96d8b5f7-3561-44a1-86a4-270fb0d1d8a6)
|
|
96
99
|
|
|
@@ -103,6 +106,13 @@ Most eval tools stop at detect and compare. EvalView helps you classify changes,
|
|
|
103
106
|
- Separate provider/model drift from real system regressions
|
|
104
107
|
- Auto-heal flaky failures with retries, review gates, and audit logs
|
|
105
108
|
|
|
109
|
+
Built for **frontier-lab rigor, startup-team practicality**:
|
|
110
|
+
- targeted behavior runs instead of giant always-on eval suites
|
|
111
|
+
- deterministic diffs first, LLM judgment where it adds signal
|
|
112
|
+
- faster loops from change -> eval -> review -> ship
|
|
113
|
+
|
|
114
|
+
[How we run EvalView with this operating model →](docs/OPERATING_MODEL.md)
|
|
115
|
+
|
|
106
116
|
```
|
|
107
117
|
✓ login-flow PASSED
|
|
108
118
|
⚠ refund-request TOOLS_CHANGED
|
|
@@ -112,6 +122,32 @@ Most eval tools stop at detect and compare. EvalView helps you classify changes,
|
|
|
112
122
|
Score: 85 → 55 Output similarity: 35%
|
|
113
123
|
```
|
|
114
124
|
|
|
125
|
+
The money screen is the one-line **verdict** that lands under every check — a single ship/don't-ship decision derived from the diff, quarantine state, cost delta, and drift confidence:
|
|
126
|
+
|
|
127
|
+
```
|
|
128
|
+
─────────────────────────────────────────────
|
|
129
|
+
VERDICT: 🛑 BLOCK RELEASE
|
|
130
|
+
─────────────────────────────────────────────
|
|
131
|
+
|
|
132
|
+
• 1 regression: billing-dispute
|
|
133
|
+
• 1 test changed behavior: refund-request
|
|
134
|
+
• Cost up 14% vs baseline
|
|
135
|
+
|
|
136
|
+
Likely cause & next actions:
|
|
137
|
+
|
|
138
|
+
1. Rerun statistically to distinguish flake from real drift
|
|
139
|
+
(high severity, high confidence)
|
|
140
|
+
→ evalview check --statistical 5
|
|
141
|
+
|
|
142
|
+
2. Review tool descriptions for: escalate_to_human
|
|
143
|
+
(high severity, high confidence)
|
|
144
|
+
Tool selection changed — usually a prompt edit nudged the model
|
|
145
|
+
→ evalview replay refund-request --trace
|
|
146
|
+
→ evalview golden update refund-request # if the new path is correct
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
Four tiers: `SAFE_TO_SHIP`, `SHIP_WITH_QUARANTINE`, `INVESTIGATE`, `BLOCK_RELEASE`. The verdict is part of `--json` output, the PR comment, and the cloud ship page — CLI, CI, and dashboard all tell the same story.
|
|
150
|
+
|
|
115
151
|
## Quick Start
|
|
116
152
|
|
|
117
153
|
```bash
|
|
@@ -126,6 +162,28 @@ evalview check # Catch regressions after every change
|
|
|
126
162
|
|
|
127
163
|
That's it. Three commands to regression-test any AI agent. `init` auto-detects your agent type (chat, tool-use, multi-step, RAG, coding) and configures the right evaluators, thresholds, and assertions.
|
|
128
164
|
|
|
165
|
+
**After `check`, the investigative loop:**
|
|
166
|
+
|
|
167
|
+
```bash
|
|
168
|
+
evalview since # 2-second brief: what's changed since your last run
|
|
169
|
+
evalview progress --since yesterday # delta report: what improved/regressed
|
|
170
|
+
evalview drift # per-test sparklines + incident markers
|
|
171
|
+
evalview slack-digest --webhook $SLACK # post the daily verdict to your team
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
These four commands turn a red ✗ into an answer — *is it real drift, a known flake, or a provider update?* — before anyone opens Slack. `since` is the habit anchor (daily open-the-terminal glance); `progress` answers "did my changes help?" with a worth-a-commit gate; `drift` visualizes the trend; `slack-digest` broadcasts the verdict. See [Daily Workflow →](#daily-workflow).
|
|
175
|
+
|
|
176
|
+
### Catch silent drift in closed models
|
|
177
|
+
|
|
178
|
+
Worried that `claude-opus-4-5` might behave differently next week without warning? [`evalview model-check`](#model-drift-detection) runs a zero-judge canary suite directly against the provider and tells you exactly when the model has drifted — no agent required, no calibration.
|
|
179
|
+
|
|
180
|
+
```bash
|
|
181
|
+
evalview model-check --model claude-opus-4-5-20251101 # first run saves baseline
|
|
182
|
+
evalview model-check --model claude-opus-4-5-20251101 # next week, detects any change
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
[See the full model-check section →](#model-drift-detection)
|
|
186
|
+
|
|
129
187
|
<details>
|
|
130
188
|
<summary><strong>Other install methods</strong></summary>
|
|
131
189
|
|
|
@@ -165,6 +223,111 @@ evalview init --profile rag # Override auto-detect
|
|
|
165
223
|
|
|
166
224
|
</details>
|
|
167
225
|
|
|
226
|
+
## Daily Workflow
|
|
227
|
+
|
|
228
|
+
Detection is only the first step. EvalView gives you the full investigative loop — so when a test goes red, you can answer *"is it real drift, a known flake, or a provider update?"* in four commands, before anyone opens Slack.
|
|
229
|
+
|
|
230
|
+
**Open the terminal — `evalview since`**
|
|
231
|
+
|
|
232
|
+
```
|
|
233
|
+
╭──────────────────────────── Since your last check ─────────────────────────────╮
|
|
234
|
+
│ │
|
|
235
|
+
│ 95% pass rate across 14 runs │
|
|
236
|
+
│ ⚠ 2 soft change(s) │
|
|
237
|
+
│ ✨ improved: search_cases, summarize_thread │
|
|
238
|
+
│ │
|
|
239
|
+
╰────────────────────────────────────────────────────────────────────────────────╯
|
|
240
|
+
|
|
241
|
+
Drift sparklines (most-declining first)
|
|
242
|
+
▇▆▅▄▃▂▁▂ billing-dispute
|
|
243
|
+
|
|
244
|
+
🔥 Streak: 6 days of clean merges
|
|
245
|
+
|
|
246
|
+
One thing to look at first:
|
|
247
|
+
→ evalview replay billing-dispute
|
|
248
|
+
```
|
|
249
|
+
|
|
250
|
+
`evalview since` is the **2-second habit brief** — one hero number, one concern, one action. It reuses Week 1's fingerprinted history, so the "since your last check" window is accurate whether you ran it 10 minutes ago or 4 days ago. Night owls at 2am and daily shippers at 9am see the same command; the label adapts. It's the command that goes in your `.zshrc` so `evalview since` fires when you open the terminal, before the espresso machine is warm.
|
|
251
|
+
|
|
252
|
+
**Morning — `evalview progress --since yesterday`**
|
|
253
|
+
|
|
254
|
+
```
|
|
255
|
+
✨ 3 test(s) now passing that weren't
|
|
256
|
+
⚠ 1 test(s) regressed
|
|
257
|
+
|
|
258
|
+
Improved:
|
|
259
|
+
+ refund-flow
|
|
260
|
+
+ order-lookup (at a4f2e91)
|
|
261
|
+
|
|
262
|
+
Regressed:
|
|
263
|
+
− billing-dispute
|
|
264
|
+
|
|
265
|
+
Output similarity: 85.20% → 87.50% ↑ +2.30%
|
|
266
|
+
|
|
267
|
+
Worth a commit:
|
|
268
|
+
✓ refund-flow (high confidence)
|
|
269
|
+
→ evalview golden update refund-flow
|
|
270
|
+
```
|
|
271
|
+
|
|
272
|
+
A **"worth a commit" gate** (3+ consecutive passes) keeps you from celebrating flakes at 2am.
|
|
273
|
+
|
|
274
|
+
**Triage — `evalview drift billing-dispute`**
|
|
275
|
+
|
|
276
|
+
```
|
|
277
|
+
Test │ Trend │ Samples │ Slope │ First → Last │ Status
|
|
278
|
+
─────────────────┼──────────────┼─────────┼────────┼──────────────┼────────
|
|
279
|
+
billing-dispute │ ! │ 20 │ -1.5% │ 90% → 80% │ declining
|
|
280
|
+
│ ▇▆▅▄▃▂▁▂▁▂▁ │ │ │ │
|
|
281
|
+
|
|
282
|
+
Most concerning: billing-dispute — slope -1.50% per check over 20 samples
|
|
283
|
+
→ evalview replay billing-dispute --trace
|
|
284
|
+
```
|
|
285
|
+
|
|
286
|
+
Unicode sparklines + OLS slope + **incident markers** (`!`) show *when* the test flipped. Drift is graded `insufficient_history / stable / low / medium / high` — not a binary alarm.
|
|
287
|
+
|
|
288
|
+
**Verdict — `evalview check --statistical 5`**
|
|
289
|
+
|
|
290
|
+
When the verdict layer returns `INVESTIGATE`, a stability-replay recommendation is auto-injected at position #0 of the action list, surviving the severity sort so you never miss it.
|
|
291
|
+
|
|
292
|
+
**Quarantine — `evalview quarantine list`**
|
|
293
|
+
|
|
294
|
+
```
|
|
295
|
+
Test │ Owner │ Age │ Flaky │ Trend │ Status │ Reason
|
|
296
|
+
──────────────────┼─────────┼──────┼───────┼───────┼───────────┼──────────────
|
|
297
|
+
race-condition │ @hidai │ 12d │ 4 │ ↘ │ ⏸ active │ race condition
|
|
298
|
+
db-timeout │ @jane │ 45d │ 8 │ ↗ │ ⏰ STALE │ db timeout
|
|
299
|
+
|
|
300
|
+
⏰ 1 entry stale — review overdue.
|
|
301
|
+
Either fix the underlying flake or remove from quarantine:
|
|
302
|
+
evalview quarantine remove db-timeout
|
|
303
|
+
```
|
|
304
|
+
|
|
305
|
+
Known-flaky tests **don't block CI** — but staleness tracking, owner tags, and a flaky-count trend glyph keep the list honest. Governance built in, not a dumping ground.
|
|
306
|
+
|
|
307
|
+
**Broadcast — `evalview slack-digest --webhook $SLACK_WEBHOOK`**
|
|
308
|
+
|
|
309
|
+
```
|
|
310
|
+
📊 EvalView digest — yesterday
|
|
311
|
+
🟢 95% pass rate across 47 runs
|
|
312
|
+
|
|
313
|
+
Drift
|
|
314
|
+
▇▆▅ billing-dispute
|
|
315
|
+
|
|
316
|
+
⏰ Stale quarantine
|
|
317
|
+
1 overdue
|
|
318
|
+
• db-timeout — @jane — 45d
|
|
319
|
+
|
|
320
|
+
🎯 Next: evalview check --fail-on REGRESSION
|
|
321
|
+
|
|
322
|
+
✓ Digest posted to Slack.
|
|
323
|
+
```
|
|
324
|
+
|
|
325
|
+
Stdlib-only Block Kit post (zero new deps). Fails soft on bad webhooks. Ends with **one actionable command** your team can copy-paste from the channel.
|
|
326
|
+
|
|
327
|
+
---
|
|
328
|
+
|
|
329
|
+
**The loop closes:** detection → investigation → graded verdict → quarantine governance → broadcast. You wake up, run `progress`, triage with `drift`, confirm with `check --statistical`, and the team sees the digest before standup. That's the morning ritual — reach for it before the espresso machine warms up.
|
|
330
|
+
|
|
168
331
|
## Why EvalView?
|
|
169
332
|
|
|
170
333
|
Use LangSmith for observability. Use Braintrust for scoring. **Use EvalView for regression gating.**
|
|
@@ -190,6 +353,9 @@ Use LangSmith for observability. Use Braintrust for scoring. **Use EvalView for
|
|
|
190
353
|
| ⚠️ **TOOLS_CHANGED** | Different tools called | Review the diff |
|
|
191
354
|
| ⚠️ **OUTPUT_CHANGED** | Same tools, output shifted | Review the diff |
|
|
192
355
|
| ❌ **REGRESSION** | Score dropped significantly | Fix before shipping |
|
|
356
|
+
| 📉 **DRIFTING** | Trend sliding with graded confidence (low/med/high) | Run `evalview drift <test>` |
|
|
357
|
+
| 🔎 **INVESTIGATE** | Verdict layer wants statistical replay | Run `evalview check --statistical 5` |
|
|
358
|
+
| ⏳ **QUARANTINED** | Known-flaky, excluded from CI exit code | Fix underlying flake or remove |
|
|
193
359
|
|
|
194
360
|
### Model / Runtime Change Detection
|
|
195
361
|
|
|
@@ -197,7 +363,7 @@ EvalView does more than compare `model_id`.
|
|
|
197
363
|
|
|
198
364
|
- **Declared model change**: adapter-reported model changed from baseline
|
|
199
365
|
- **Runtime fingerprint change**: observed model labels in the trace changed, even when the top-level model name is missing
|
|
200
|
-
- **Coordinated drift**: multiple tests shift together in the same check run, which often points to a silent provider rollout or runtime change
|
|
366
|
+
- **Coordinated drift**: multiple tests shift together in the same check run, which often points to a silent provider rollout or runtime change — now **graded `low / medium / high`** via `DriftTracker.classify_drift`, not a binary alarm
|
|
201
367
|
|
|
202
368
|
When detected, `evalview check` surfaces a run-level signal with a classification (`declared` or `suspected`), confidence level, and evidence from fingerprints, retries, and affected tests.
|
|
203
369
|
|
|
@@ -461,9 +627,13 @@ Works with **LangGraph, CrewAI, OpenAI, Claude, Mistral, HuggingFace, Ollama, MC
|
|
|
461
627
|
|
|
462
628
|
1. **`evalview init`** — detects your running agent, creates a starter test suite
|
|
463
629
|
2. **`evalview snapshot`** — runs tests, saves traces as baselines
|
|
464
|
-
3. **`evalview check`** — replays tests, diffs against baselines, opens HTML report
|
|
465
|
-
4. **`evalview
|
|
466
|
-
5. **`evalview
|
|
630
|
+
3. **`evalview check`** — replays tests, diffs against baselines, emits the ship/don't-ship verdict, opens HTML report
|
|
631
|
+
4. **`evalview since`** — 2-second brief: what's changed since your last run (the daily habit anchor)
|
|
632
|
+
5. **`evalview watch`** — re-runs checks on every file save
|
|
633
|
+
6. **`evalview monitor`** — continuous checks in production with Slack alerts
|
|
634
|
+
7. **`evalview progress --since`** — diff any two points in history with a "worth a commit" gate
|
|
635
|
+
8. **`evalview drift`** — per-test sparklines, OLS slope, and incident markers
|
|
636
|
+
9. **`evalview slack-digest`** — post the daily verdict to your team channel
|
|
467
637
|
|
|
468
638
|
<details>
|
|
469
639
|
<summary><strong>Snapshot management</strong></summary>
|
|
@@ -488,16 +658,141 @@ evalview monitor # Check every 5 min
|
|
|
488
658
|
evalview monitor --dashboard # Live terminal dashboard
|
|
489
659
|
evalview monitor --slack-webhook https://hooks.slack.com/services/...
|
|
490
660
|
evalview monitor --history monitor.jsonl # JSONL for dashboards
|
|
661
|
+
evalview monitor --incidents # Log confirmed regressions for `evalview autopr`
|
|
491
662
|
```
|
|
492
663
|
|
|
493
664
|
New regressions trigger Slack alerts. Recoveries send all-clear. No spam on persistent failures.
|
|
494
665
|
|
|
666
|
+
**Every alert is a promise.** The monitor requires **two consecutive failing cycles** before it pages a human — a single blip self-resolves silently and never interrupts anyone. If a test must alert on the first failure (auth, payments, PII, refund paths), mark it `gate: strict` in its YAML and it bypasses the gate, re-alerting every cycle until it passes.
|
|
667
|
+
|
|
668
|
+
Suppressed failures are never hidden: `evalview slack-digest` renders a Noise section listing every test the gate swallowed, how many times it self-resolved, and a visible false-positive rate (`3 suppressed / 12 fired = 25% noise`). See [`evalview/core/noise_tracker.py`](evalview/core/noise_tracker.py) for the full design — confirmation gate, coordinated-incident collapse, and the `.evalview/noise.jsonl` metric.
|
|
669
|
+
|
|
495
670
|
[Monitor config options →](docs/CLI_REFERENCE.md)
|
|
496
671
|
|
|
672
|
+
## Auto-PR Regression Tests From Production Incidents
|
|
673
|
+
|
|
674
|
+
`evalview autopr` closes the loop: **production failure → pinned regression test → pull request**, with zero LLM calls and zero manual YAML writing.
|
|
675
|
+
|
|
676
|
+
```bash
|
|
677
|
+
evalview monitor --incidents # Monitor writes .evalview/incidents.jsonl
|
|
678
|
+
evalview autopr --dry-run # Preview what would be generated
|
|
679
|
+
evalview autopr # Write tests/regressions/*.yaml
|
|
680
|
+
evalview autopr --open-pr # + commit + push + gh pr create
|
|
681
|
+
```
|
|
682
|
+
|
|
683
|
+
The synthesizer is **pure and deterministic** — no network, no LLM — so it runs instantly in CI. For each confirmed regression in `.evalview/incidents.jsonl` it:
|
|
684
|
+
|
|
685
|
+
- builds a `tests/regressions/<slug>.yaml` that pins the query, the baseline tool sequence (`expected.tools`), the newly-appeared tools (`forbidden_tools`), and short phrases from the bad output (`output.not_contains`)
|
|
686
|
+
- tags the test `suite_type: regression`, `gate: strict`, and stamps the incident metadata into `meta.incident` so `autopr` can skip it on subsequent runs
|
|
687
|
+
- defaults to `min_score: 90` — regression tests are a safety net, not a capability benchmark
|
|
688
|
+
|
|
689
|
+
```yaml
|
|
690
|
+
# tests/regressions/refund-request-b3c4d5e6.yaml (auto-generated)
|
|
691
|
+
name: regression_refund-request_2026-04-14
|
|
692
|
+
description: Auto-generated from production incident (REGRESSION) at 2026-04-14T12:34:56Z ...
|
|
693
|
+
input:
|
|
694
|
+
query: "I want a refund for order #123"
|
|
695
|
+
expected:
|
|
696
|
+
tools: [lookup_order, check_policy, process_refund]
|
|
697
|
+
forbidden_tools: [escalate_to_human] # appeared only in the failing trace
|
|
698
|
+
output:
|
|
699
|
+
not_contains: ["Sure, I've processed your refund for $999."]
|
|
700
|
+
thresholds:
|
|
701
|
+
min_score: 90.0
|
|
702
|
+
suite_type: regression
|
|
703
|
+
gate: strict
|
|
704
|
+
tags: [incident, autopr]
|
|
705
|
+
```
|
|
706
|
+
|
|
707
|
+
**Wire it into GitHub Actions:** copy [`examples/github-workflow-autopr.yml`](examples/github-workflow-autopr.yml) to `.github/workflows/evalview-autopr.yml` — the workflow runs `monitor` then `autopr --open-pr --require-new` on a schedule. Every production regression arrives as a reviewable PR, and your hallucination test suite grows by itself.
|
|
708
|
+
|
|
709
|
+
**The CLI is fully local.** `evalview monitor` + `evalview autopr` run entirely on your machine — local files, `gh pr create`, no network calls, no cloud account required. The primitive is free and open-source forever.
|
|
710
|
+
|
|
711
|
+
## Model Drift Detection
|
|
712
|
+
|
|
713
|
+
Closed models update silently. `evalview model-check` is a dedicated command that runs a fixed structural canary suite directly against the provider and tells you when the model itself has changed — no agent, no LLM judge, no calibration required.
|
|
714
|
+
|
|
715
|
+
```bash
|
|
716
|
+
# Save a baseline snapshot the day you deploy
|
|
717
|
+
evalview model-check --model claude-opus-4-5-20251101
|
|
718
|
+
|
|
719
|
+
# Run it weekly — detects any behavioral change against that baseline
|
|
720
|
+
evalview model-check --model claude-opus-4-5-20251101
|
|
721
|
+
```
|
|
722
|
+
|
|
723
|
+
**Example output when drift is detected:**
|
|
724
|
+
|
|
725
|
+
```
|
|
726
|
+
EvalView model-check
|
|
727
|
+
Model: claude-opus-4-5-20251101
|
|
728
|
+
Provider: anthropic
|
|
729
|
+
Suite: canary v1.public (15 prompts, sha256:6b8e925a5543…)
|
|
730
|
+
Runs/prompt: 1
|
|
731
|
+
Temperature: 0.0
|
|
732
|
+
Cost: $0.0228
|
|
733
|
+
|
|
734
|
+
vs reference (2026-04-10, 7d ago)
|
|
735
|
+
Drift: MODEL (MEDIUM confidence)
|
|
736
|
+
Pass rate: 15/15 → 13/15 (-13.3%)
|
|
737
|
+
Flipped: tool_choice_refund_first_step, json_schema_order_summary
|
|
738
|
+
|
|
739
|
+
vs previous (2026-04-17, 0d ago)
|
|
740
|
+
Drift: NONE
|
|
741
|
+
Pass rate: 13/15 → 13/15 (+0.0%)
|
|
742
|
+
```
|
|
743
|
+
|
|
744
|
+
**How it works:**
|
|
745
|
+
|
|
746
|
+
| Check type | What it catches |
|
|
747
|
+
|------------|-----------------|
|
|
748
|
+
| **Tool choice** | Did the model pick the right tool, in the right order? |
|
|
749
|
+
| **JSON schema** | Does the output still match the expected structure? |
|
|
750
|
+
| **Refusal** | Did the model refuse when it should (or comply when it should)? |
|
|
751
|
+
| **Exact match** | Does the response match a regex anchor? |
|
|
752
|
+
|
|
753
|
+
Every check runs at `temperature=0` for determinism. Drift is classified as **NONE / WEAK / MEDIUM / STRONG** based on how many prompts flipped pass↔fail. No judge — the signal is structural, not probabilistic.
|
|
754
|
+
|
|
755
|
+
```bash
|
|
756
|
+
evalview model-check --model claude-opus-4-5-20251101 --dry-run # Cost estimate before running
|
|
757
|
+
evalview model-check --model claude-opus-4-5-20251101 --pin # Pin this run as the new reference
|
|
758
|
+
evalview model-check --model claude-opus-4-5-20251101 --reset-reference # Start a fresh baseline
|
|
759
|
+
evalview model-check --model claude-opus-4-5-20251101 --json # Machine-readable output for CI
|
|
760
|
+
evalview model-check --model claude-opus-4-5-20251101 --suite my-canary.yaml # Bring your own suite
|
|
761
|
+
```
|
|
762
|
+
|
|
763
|
+
Ships with a **bundled 15-prompt public canary** covering tool selection, JSON schema, refusal behavior, and exact match. Add your own prompts with `--suite`. v1 supports **Anthropic**; OpenAI/Mistral/Cohere land in v1.1.
|
|
764
|
+
|
|
765
|
+
[Full reference → docs/MODEL_CHECK.md](docs/MODEL_CHECK.md)
|
|
766
|
+
|
|
767
|
+
## Simulation + Decision Rationale
|
|
768
|
+
|
|
769
|
+
Two features that close the gaps the April 2026 agent-eval reports kept calling out: pre-flight what-if testing and structured "why did the agent branch?" logging.
|
|
770
|
+
|
|
771
|
+
**Pre-flight simulation** — `evalview simulate` runs your test suite hermetically against declared mocks (tool calls, LLM responses, HTTP). Deterministic seed, zero cost, works in CI. Declare mocks in the test YAML, fan out with `--variants N`:
|
|
772
|
+
|
|
773
|
+
```bash
|
|
774
|
+
evalview simulate tests/ --variants 5 --seed 42
|
|
775
|
+
```
|
|
776
|
+
|
|
777
|
+
**Decision rationale** — every tool_choice / branch gets recorded with the chosen option, alternatives considered, and any model-reported reasoning (Anthropic `thinking` blocks auto-captured). Grouped across runs by `input_hash` so cloud analytics surfaces decision drift before your users notice. Local HTML replay shows it inline.
|
|
778
|
+
|
|
779
|
+
Supported adapters: Anthropic, OpenAI Assistants, LangGraph, CrewAI native.
|
|
780
|
+
|
|
781
|
+
[`docs/SIMULATE.md`](docs/SIMULATE.md) · [`docs/RATIONALE.md`](docs/RATIONALE.md) · [`examples/simulation/`](examples/simulation)
|
|
782
|
+
|
|
497
783
|
## Key Features
|
|
498
784
|
|
|
499
785
|
| Feature | Description | Docs |
|
|
500
786
|
|---------|-------------|------|
|
|
787
|
+
| **Progress command** | `evalview progress --since <date\|sha>` — improved/regressed with "worth a commit" gate | [Above](#daily-workflow) |
|
|
788
|
+
| **Drift command** | `evalview drift` — unicode sparklines, OLS slope, incident markers | [Above](#daily-workflow) |
|
|
789
|
+
| **Slack digest** | `evalview slack-digest` — stdlib Block Kit post with one actionable next-step | [Above](#daily-workflow) |
|
|
790
|
+
| **Flake quarantine** | Known-flaky tests don't block CI; staleness tracking, owner tags, governance | [Above](#daily-workflow) |
|
|
791
|
+
| **Release verdict layer** | Graded drift confidence + auto-injected stability recommendation | [Above](#daily-workflow) |
|
|
792
|
+
| **Recommendation engine** | Suggests the next command from verdict, drift class, and history | [Above](#daily-workflow) |
|
|
793
|
+
| **Model drift detection** | `model-check` — zero-judge canary suite that catches silent model updates | [Docs](#model-drift-detection) |
|
|
794
|
+
| **Simulation harness** | `evalview simulate` — hermetic what-if runs against declared mocks, with `--variants N` fan-out | [Docs](docs/SIMULATE.md) |
|
|
795
|
+
| **Decision rationale** | Structured `tool_choice` / `branch` logging with cross-run grouping for decision-drift detection | [Docs](docs/RATIONALE.md) |
|
|
501
796
|
| **Assertion wizard** | Analyze captured traffic, suggest smart assertions automatically | [Above](#assertion-wizard--tests-from-real-traffic) |
|
|
502
797
|
| **Auto-variant discovery** | Run N times, cluster paths, save valid variants | [Above](#auto-variant-discovery--solve-non-determinism) |
|
|
503
798
|
| **Auto-heal** | Retry flakes, propose variants, escalate structural changes | [Above](#auto-heal--fix-flakes-without-leaving-ci) |
|
|
@@ -508,6 +803,7 @@ New regressions trigger Slack alerts. Recoveries send all-clear. No spam on pers
|
|
|
508
803
|
| **Multi-reference baselines** | Up to 5 variants for non-deterministic agents | [Docs](docs/GOLDEN_TRACES.md) |
|
|
509
804
|
| **`forbidden_tools`** | Safety contracts — hard-fail on any violation | [Docs](docs/YAML_SCHEMA.md) |
|
|
510
805
|
| **Watch mode** | `evalview watch` — re-run checks on file save, with dashboard | [Docs](#watch-mode) |
|
|
806
|
+
| **Model comparison** | `run_eval` / `compare_models` — test one query across N models in parallel | [Docs](#model-comparison) |
|
|
511
807
|
| **Python API** | `gate()` / `gate_async()` — programmatic regression checks | [Docs](#python-api) |
|
|
512
808
|
| **PR comments + alerts** | Cost/latency spikes, model changes, collapsible diffs | [Docs](docs/CI_CD.md) |
|
|
513
809
|
| **Terminal dashboard** | Scorecard, sparkline trends, confidence scoring | — |
|
|
@@ -520,6 +816,7 @@ New regressions trigger Slack alerts. Recoveries send all-clear. No spam on pers
|
|
|
520
816
|
| **Multi-turn capture** | `capture --multi-turn` records conversations as tests | [Docs](#multi-turn-testing) |
|
|
521
817
|
| **Semantic similarity** | Embedding-based output comparison | [Docs](docs/EVALUATION_METRICS.md) |
|
|
522
818
|
| **Production monitoring** | `evalview monitor --dashboard` with Slack alerts and JSONL history | [Docs](#production-monitoring) |
|
|
819
|
+
| **Auto-PR regression tests** | `evalview autopr` turns `.evalview/incidents.jsonl` into pinned regression tests + PRs | [Docs](#auto-pr-regression-tests-from-production-incidents) |
|
|
523
820
|
| **A/B comparison** | `evalview compare --v1 <url> --v2 <url>` | [Docs](docs/CLI_REFERENCE.md) |
|
|
524
821
|
| **Test generation** | `evalview generate` — discovers your agent's domain, generates relevant tests | [Docs](docs/TEST_GENERATION.md) |
|
|
525
822
|
| **Per-turn judge scoring** | Multi-turn output quality scored per turn with conversation context | [Docs](#multi-turn-testing) |
|
|
@@ -530,6 +827,7 @@ New regressions trigger Slack alerts. Recoveries send all-clear. No spam on pers
|
|
|
530
827
|
| **Verified cost tracking** | Per-test cost breakdown with model pricing rates | [Docs](docs/COST_TRACKING.md) |
|
|
531
828
|
| **Judge model picker** | Choose GPT, Claude, Gemini, DeepSeek, or Ollama (free) | [Docs](docs/EVALUATION_METRICS.md) |
|
|
532
829
|
| **Pytest plugin** | `evalview_check` fixture for standard pytest | [Docs](#pytest-plugin) |
|
|
830
|
+
| **Model comparison** | `run_eval` / `compare_models` — parametrize tests across models, auto-detect provider | [Docs](#model-comparison) |
|
|
533
831
|
| **GitHub Actions job summary** | Results visible in Actions UI, not just PR comments | [Docs](docs/CI_CD.md) |
|
|
534
832
|
| **Git hooks** | Pre-push regression blocking, zero CI config | [Docs](docs/CI_CD.md) |
|
|
535
833
|
| **LLM judge caching** | ~80% cost reduction in statistical mode | [Docs](docs/EVALUATION_METRICS.md) |
|
|
@@ -618,6 +916,71 @@ pip install evalview # Plugin registers automatically
|
|
|
618
916
|
pytest # Runs alongside your existing tests
|
|
619
917
|
```
|
|
620
918
|
|
|
919
|
+
## Model Comparison
|
|
920
|
+
|
|
921
|
+
Test the same task across multiple models with one parametrized test. No config files — just a model name and a query.
|
|
922
|
+
|
|
923
|
+
```python
|
|
924
|
+
import pytest
|
|
925
|
+
import evalview
|
|
926
|
+
|
|
927
|
+
@pytest.mark.parametrize("model", ["claude-opus-4-6", "gpt-4o", "claude-sonnet-4-6"])
|
|
928
|
+
def test_my_task(model):
|
|
929
|
+
result = evalview.run_eval(model, query="Summarize this contract in one sentence.")
|
|
930
|
+
assert evalview.score(result) > 0.8
|
|
931
|
+
```
|
|
932
|
+
|
|
933
|
+
Provider is auto-detected from the model name. Requires `ANTHROPIC_API_KEY` / `OPENAI_API_KEY` depending on which models you use.
|
|
934
|
+
|
|
935
|
+
**Score against expected output** — token-overlap similarity, no LLM judge needed:
|
|
936
|
+
|
|
937
|
+
```python
|
|
938
|
+
result = evalview.run_eval(
|
|
939
|
+
"gpt-4o",
|
|
940
|
+
query="What language is Python?",
|
|
941
|
+
expected="Python is a high-level interpreted language.",
|
|
942
|
+
threshold=0.4,
|
|
943
|
+
)
|
|
944
|
+
```
|
|
945
|
+
|
|
946
|
+
**Custom scorer** — assert specific behavior:
|
|
947
|
+
|
|
948
|
+
```python
|
|
949
|
+
def has_json(output, expected):
|
|
950
|
+
import json, re
|
|
951
|
+
m = re.search(r"\{.*?\}", output, re.DOTALL)
|
|
952
|
+
try: return 1.0 if json.loads(m.group()) else 0.0
|
|
953
|
+
except: return 0.0
|
|
954
|
+
|
|
955
|
+
result = evalview.run_eval("claude-opus-4-6", query="Return JSON: {name, age}", scorer=has_json)
|
|
956
|
+
assert evalview.score(result) == 1.0
|
|
957
|
+
```
|
|
958
|
+
|
|
959
|
+
**Run all models in parallel and compare:**
|
|
960
|
+
|
|
961
|
+
```python
|
|
962
|
+
results = evalview.compare_models(
|
|
963
|
+
query="Explain quantum entanglement in one sentence.",
|
|
964
|
+
models=["claude-opus-4-6", "gpt-4o", "claude-sonnet-4-6"],
|
|
965
|
+
)
|
|
966
|
+
evalview.print_comparison_table(results) # Rich table: score, latency, cost
|
|
967
|
+
best = results[0] # sorted best-first
|
|
968
|
+
```
|
|
969
|
+
|
|
970
|
+
```
|
|
971
|
+
┏━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━┓
|
|
972
|
+
┃ Model ┃ Score ┃ Latency ┃ Cost ┃ Pass? ┃
|
|
973
|
+
┡━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━┩
|
|
974
|
+
│ claude-opus-4-6 │ 1.00 │ 842ms │ $0.00312 │ ✓ │
|
|
975
|
+
│ gpt-4o │ 1.00 │ 631ms │ $0.00087 │ ✓ │
|
|
976
|
+
│ claude-sonnet-4-6 │ 1.00 │ 514ms │ $0.00063 │ ✓ │
|
|
977
|
+
└────────────────────┴───────┴──────────┴───────────┴───────┘
|
|
978
|
+
```
|
|
979
|
+
|
|
980
|
+
`ModelResult` fields: `.model`, `.output`, `.score`, `.latency_ms`, `.cost_usd`, `.passed`, `.error`
|
|
981
|
+
|
|
982
|
+
[Full example →](examples/model_comparison_test.py)
|
|
983
|
+
|
|
621
984
|
## Claude Code (MCP)
|
|
622
985
|
|
|
623
986
|
```bash
|
|
@@ -662,6 +1025,7 @@ Then just ask Claude: "did my refactor break anything?" and it runs `run_check`
|
|
|
662
1025
|
|---|---|---|
|
|
663
1026
|
| [Getting Started](docs/GETTING_STARTED.md) | [Golden Traces](docs/GOLDEN_TRACES.md) | [CI/CD](docs/CI_CD.md) |
|
|
664
1027
|
| [CLI Reference](docs/CLI_REFERENCE.md) | [Evaluation Metrics](docs/EVALUATION_METRICS.md) | [MCP Contracts](docs/MCP_CONTRACTS.md) |
|
|
1028
|
+
| [Simulation](docs/SIMULATE.md) | [Decision Rationale](docs/RATIONALE.md) | |
|
|
665
1029
|
| [Agent Instructions](AGENT_INSTRUCTIONS.md) | [Agent Recipes](docs/agent-recipes/README.md) | [Ollama Recipe](docs/agent-recipes/integrate-ollama.md) |
|
|
666
1030
|
| [FAQ](docs/FAQ.md) | [Test Generation](docs/TEST_GENERATION.md) | [Skills Testing](docs/SKILLS_TESTING.md) |
|
|
667
1031
|
| [YAML Schema](docs/YAML_SCHEMA.md) | [Statistical Mode](docs/STATISTICAL_MODE.md) | [Chat Mode](docs/CHAT_MODE.md) |
|