evalview 0.6.1__tar.gz → 0.7.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (309) hide show
  1. {evalview-0.6.1/evalview.egg-info → evalview-0.7.0}/PKG-INFO +371 -7
  2. {evalview-0.6.1 → evalview-0.7.0}/README.md +369 -6
  3. evalview-0.7.0/evalview/__init__.py +15 -0
  4. evalview-0.7.0/evalview/adapters/aider_adapter.py +535 -0
  5. {evalview-0.6.1 → evalview-0.7.0}/evalview/adapters/anthropic_adapter.py +33 -0
  6. {evalview-0.6.1 → evalview-0.7.0}/evalview/adapters/crewai_native_adapter.py +34 -0
  7. {evalview-0.6.1 → evalview-0.7.0}/evalview/adapters/langgraph_adapter.py +47 -2
  8. {evalview-0.6.1 → evalview-0.7.0}/evalview/adapters/openai_assistants_adapter.py +18 -0
  9. evalview-0.7.0/evalview/adapters/opencode_adapter.py +387 -0
  10. {evalview-0.6.1 → evalview-0.7.0}/evalview/adapters/registry.py +14 -0
  11. {evalview-0.6.1 → evalview-0.7.0}/evalview/api.py +13 -1
  12. evalview-0.7.0/evalview/benchmarks/canary/README.md +93 -0
  13. evalview-0.7.0/evalview/benchmarks/canary/__init__.py +18 -0
  14. evalview-0.7.0/evalview/benchmarks/canary/suite.v1.held-out.yaml +66 -0
  15. evalview-0.7.0/evalview/benchmarks/canary/suite.v1.public.yaml +214 -0
  16. {evalview-0.6.1 → evalview-0.7.0}/evalview/chat.py +23 -7
  17. {evalview-0.6.1 → evalview-0.7.0}/evalview/ci/comment.py +255 -2
  18. {evalview-0.6.1 → evalview-0.7.0}/evalview/cli.py +37 -1
  19. evalview-0.7.0/evalview/cloud/push.py +328 -0
  20. evalview-0.7.0/evalview/commands/autopr_cmd.py +526 -0
  21. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/check_cmd.py +528 -17
  22. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/check_display.py +318 -18
  23. evalview-0.7.0/evalview/commands/drift_cmd.py +335 -0
  24. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/listing_cmd.py +4 -6
  25. evalview-0.7.0/evalview/commands/log_cmd.py +212 -0
  26. evalview-0.7.0/evalview/commands/model_check_cmd.py +691 -0
  27. evalview-0.7.0/evalview/commands/model_check_render.py +149 -0
  28. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/monitor_cmd.py +406 -115
  29. evalview-0.7.0/evalview/commands/progress_cmd.py +413 -0
  30. evalview-0.7.0/evalview/commands/quarantine_cmd.py +191 -0
  31. evalview-0.7.0/evalview/commands/replay_trace_cmd.py +257 -0
  32. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/run/_adapters.py +40 -2
  33. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/run/_cmd.py +49 -7
  34. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/run/_reporter.py +63 -5
  35. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/shared.py +119 -126
  36. evalview-0.7.0/evalview/commands/simulate_cmd.py +213 -0
  37. evalview-0.7.0/evalview/commands/since_cmd.py +536 -0
  38. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/skill_cmd.py +20 -2
  39. evalview-0.7.0/evalview/commands/slack_digest_cmd.py +423 -0
  40. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/snapshot_cmd.py +133 -59
  41. evalview-0.7.0/evalview/compare.py +486 -0
  42. {evalview-0.6.1 → evalview-0.7.0}/evalview/core/adapter_factory.py +17 -1
  43. evalview-0.7.0/evalview/core/behavioral_anomalies.py +478 -0
  44. evalview-0.7.0/evalview/core/benchmark_hardening.py +550 -0
  45. evalview-0.7.0/evalview/core/canary_suite.py +171 -0
  46. {evalview-0.6.1 → evalview-0.7.0}/evalview/core/celebrations.py +3 -0
  47. {evalview-0.6.1 → evalview-0.7.0}/evalview/core/config.py +17 -0
  48. {evalview-0.6.1 → evalview-0.7.0}/evalview/core/diff.py +8 -0
  49. evalview-0.7.0/evalview/core/discord_notifier.py +139 -0
  50. evalview-0.7.0/evalview/core/drift_classifier.py +205 -0
  51. evalview-0.7.0/evalview/core/drift_kind.py +49 -0
  52. {evalview-0.6.1 → evalview-0.7.0}/evalview/core/drift_tracker.py +168 -1
  53. evalview-0.7.0/evalview/core/model_check_scoring.py +389 -0
  54. evalview-0.7.0/evalview/core/model_provider_runner.py +224 -0
  55. evalview-0.7.0/evalview/core/model_snapshots.py +321 -0
  56. evalview-0.7.0/evalview/core/noise_tracker.py +529 -0
  57. evalview-0.7.0/evalview/core/observability.py +233 -0
  58. evalview-0.7.0/evalview/core/quarantine.py +451 -0
  59. evalview-0.7.0/evalview/core/rationale.py +226 -0
  60. evalview-0.7.0/evalview/core/recommendations.py +337 -0
  61. evalview-0.7.0/evalview/core/regression_synth.py +271 -0
  62. evalview-0.7.0/evalview/core/replay_pipeline.py +330 -0
  63. {evalview-0.6.1 → evalview-0.7.0}/evalview/core/root_cause.py +243 -8
  64. evalview-0.7.0/evalview/core/simulation.py +337 -0
  65. {evalview-0.6.1 → evalview-0.7.0}/evalview/core/slack_notifier.py +43 -5
  66. evalview-0.7.0/evalview/core/turn_coherence.py +567 -0
  67. {evalview-0.6.1 → evalview-0.7.0}/evalview/core/types.py +253 -1
  68. evalview-0.7.0/evalview/core/verdict.py +242 -0
  69. {evalview-0.6.1 → evalview-0.7.0}/evalview/evaluators/cost_evaluator.py +7 -2
  70. {evalview-0.6.1 → evalview-0.7.0}/evalview/evaluators/evaluator.py +35 -0
  71. {evalview-0.6.1 → evalview-0.7.0}/evalview/mcp_server.py +8 -0
  72. {evalview-0.6.1 → evalview-0.7.0}/evalview/pytest_plugin.py +38 -1
  73. {evalview-0.6.1 → evalview-0.7.0}/evalview/reporters/console_reporter.py +5 -1
  74. {evalview-0.6.1 → evalview-0.7.0}/evalview/reporters/html_reporter.py +293 -10
  75. {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/types.py +15 -1
  76. evalview-0.7.0/evalview/templates/patterns/code-generation.yaml +31 -0
  77. evalview-0.7.0/evalview/templates/patterns/cost-budget.yaml +20 -0
  78. evalview-0.7.0/evalview/templates/patterns/customer-support.yaml +31 -0
  79. evalview-0.7.0/evalview/templates/patterns/data-analysis.yaml +31 -0
  80. evalview-0.7.0/evalview/templates/patterns/latency-budget.yaml +20 -0
  81. evalview-0.7.0/evalview/templates/patterns/multi-turn-memory.yaml +28 -0
  82. evalview-0.7.0/evalview/templates/patterns/output-format.yaml +23 -0
  83. evalview-0.7.0/evalview/templates/patterns/rag-citation.yaml +29 -0
  84. evalview-0.7.0/evalview/templates/patterns/rag-grounding.yaml +27 -0
  85. evalview-0.7.0/evalview/templates/patterns/research-synthesis.yaml +34 -0
  86. evalview-0.7.0/evalview/templates/patterns/safety-refusal.yaml +28 -0
  87. evalview-0.7.0/evalview/templates/patterns/tool-error-handling.yaml +26 -0
  88. evalview-0.7.0/evalview/templates/patterns/tool-not-called.yaml +22 -0
  89. evalview-0.7.0/evalview/templates/patterns/tool-sequence.yaml +26 -0
  90. evalview-0.7.0/evalview/templates/patterns/wrong-tool-chosen.yaml +23 -0
  91. {evalview-0.6.1 → evalview-0.7.0}/evalview/test_generation.py +5 -1
  92. {evalview-0.6.1 → evalview-0.7.0}/evalview/visualization/generators.py +314 -10
  93. {evalview-0.6.1 → evalview-0.7.0/evalview.egg-info}/PKG-INFO +371 -7
  94. {evalview-0.6.1 → evalview-0.7.0}/evalview.egg-info/SOURCES.txt +75 -1
  95. {evalview-0.6.1 → evalview-0.7.0}/evalview.egg-info/requires.txt +1 -0
  96. {evalview-0.6.1 → evalview-0.7.0}/pyproject.toml +9 -1
  97. {evalview-0.6.1 → evalview-0.7.0}/tests/test_adapters.py +80 -0
  98. evalview-0.7.0/tests/test_autopr.py +475 -0
  99. evalview-0.7.0/tests/test_behavioral_anomalies.py +457 -0
  100. evalview-0.7.0/tests/test_benchmark_hardening.py +484 -0
  101. evalview-0.7.0/tests/test_canary_suite.py +171 -0
  102. {evalview-0.6.1 → evalview-0.7.0}/tests/test_check_cmd.py +224 -0
  103. {evalview-0.6.1 → evalview-0.7.0}/tests/test_check_pipeline.py +81 -0
  104. {evalview-0.6.1 → evalview-0.7.0}/tests/test_config_models.py +1 -0
  105. evalview-0.7.0/tests/test_drift_classifier.py +148 -0
  106. {evalview-0.6.1 → evalview-0.7.0}/tests/test_evaluator_accuracy.py +2 -2
  107. evalview-0.7.0/tests/test_model_check_cmd.py +796 -0
  108. evalview-0.7.0/tests/test_model_check_scoring.py +279 -0
  109. evalview-0.7.0/tests/test_model_snapshots.py +272 -0
  110. {evalview-0.6.1 → evalview-0.7.0}/tests/test_monitor.py +176 -1
  111. evalview-0.7.0/tests/test_noise_tracker.py +449 -0
  112. evalview-0.7.0/tests/test_observability_regressions.py +330 -0
  113. evalview-0.7.0/tests/test_rationale.py +317 -0
  114. evalview-0.7.0/tests/test_replay_pipeline.py +327 -0
  115. {evalview-0.6.1 → evalview-0.7.0}/tests/test_root_cause.py +381 -22
  116. {evalview-0.6.1 → evalview-0.7.0}/tests/test_run_cmd_tags.py +2 -2
  117. {evalview-0.6.1 → evalview-0.7.0}/tests/test_runner.py +8 -2
  118. evalview-0.7.0/tests/test_simulation.py +359 -0
  119. evalview-0.7.0/tests/test_snapshot_json_output.py +247 -0
  120. evalview-0.7.0/tests/test_turn_coherence.py +409 -0
  121. {evalview-0.6.1 → evalview-0.7.0}/tests/test_types.py +261 -0
  122. evalview-0.7.0/tests/test_verdict.py +228 -0
  123. evalview-0.7.0/tests/test_week1_integration.py +330 -0
  124. evalview-0.7.0/tests/test_week2_quarantine.py +222 -0
  125. evalview-0.7.0/tests/test_week2_review_fixes.py +284 -0
  126. evalview-0.7.0/tests/test_week2_since_and_comment.py +344 -0
  127. evalview-0.7.0/tests/test_week3_progress_and_drift.py +610 -0
  128. evalview-0.6.1/evalview/__init__.py +0 -6
  129. {evalview-0.6.1 → evalview-0.7.0}/LICENSE +0 -0
  130. {evalview-0.6.1 → evalview-0.7.0}/NOTICE +0 -0
  131. {evalview-0.6.1 → evalview-0.7.0}/evalview/__main__.py +0 -0
  132. {evalview-0.6.1 → evalview-0.7.0}/evalview/adapters/__init__.py +0 -0
  133. {evalview-0.6.1 → evalview-0.7.0}/evalview/adapters/base.py +0 -0
  134. {evalview-0.6.1 → evalview-0.7.0}/evalview/adapters/cohere_adapter.py +0 -0
  135. {evalview-0.6.1 → evalview-0.7.0}/evalview/adapters/crewai_adapter.py +0 -0
  136. {evalview-0.6.1 → evalview-0.7.0}/evalview/adapters/goose_adapter.py +0 -0
  137. {evalview-0.6.1 → evalview-0.7.0}/evalview/adapters/http_adapter.py +0 -0
  138. {evalview-0.6.1 → evalview-0.7.0}/evalview/adapters/huggingface_adapter.py +0 -0
  139. {evalview-0.6.1 → evalview-0.7.0}/evalview/adapters/mcp_adapter.py +0 -0
  140. {evalview-0.6.1 → evalview-0.7.0}/evalview/adapters/mistral_adapter.py +0 -0
  141. {evalview-0.6.1 → evalview-0.7.0}/evalview/adapters/ollama_adapter.py +0 -0
  142. {evalview-0.6.1 → evalview-0.7.0}/evalview/adapters/openclaw_adapter.py +0 -0
  143. {evalview-0.6.1 → evalview-0.7.0}/evalview/adapters/pydantic_ai_adapter.py +0 -0
  144. {evalview-0.6.1 → evalview-0.7.0}/evalview/adapters/tapescope_adapter.py +0 -0
  145. {evalview-0.6.1 → evalview-0.7.0}/evalview/benchmarks/__init__.py +0 -0
  146. {evalview-0.6.1 → evalview-0.7.0}/evalview/chat_runtime.py +0 -0
  147. {evalview-0.6.1 → evalview-0.7.0}/evalview/chat_slash.py +0 -0
  148. {evalview-0.6.1 → evalview-0.7.0}/evalview/ci/__init__.py +0 -0
  149. {evalview-0.6.1 → evalview-0.7.0}/evalview/cloud/__init__.py +0 -0
  150. {evalview-0.6.1 → evalview-0.7.0}/evalview/cloud/auth.py +0 -0
  151. {evalview-0.6.1 → evalview-0.7.0}/evalview/cloud/client.py +0 -0
  152. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/__init__.py +0 -0
  153. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/add_cmd.py +0 -0
  154. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/assertion_wizard.py +0 -0
  155. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/badge_cmd.py +0 -0
  156. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/baseline_cmd.py +0 -0
  157. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/benchmark_cmd.py +0 -0
  158. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/capture_cmd.py +0 -0
  159. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/chat_cmd.py +0 -0
  160. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/ci_cmd.py +0 -0
  161. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/cloud_cmd.py +0 -0
  162. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/demo_cmd.py +0 -0
  163. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/expand_cmd.py +0 -0
  164. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/feedback_cmd.py +0 -0
  165. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/generate_cmd.py +0 -0
  166. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/golden_cmd.py +0 -0
  167. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/gym_cmd.py +0 -0
  168. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/hooks_cmd.py +0 -0
  169. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/import_cmd.py +0 -0
  170. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/init_cmd.py +0 -0
  171. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/judge_cmd.py +0 -0
  172. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/mcp_cmd.py +0 -0
  173. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/openclaw_cmd.py +0 -0
  174. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/run/__init__.py +0 -0
  175. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/run/_executor.py +0 -0
  176. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/run/_runner.py +0 -0
  177. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/skill_agent_runner.py +0 -0
  178. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/telemetry_cmd.py +0 -0
  179. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/traces_cmd.py +0 -0
  180. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/trends_cmd.py +0 -0
  181. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/visual_cmd.py +0 -0
  182. {evalview-0.6.1 → evalview-0.7.0}/evalview/commands/watch_cmd.py +0 -0
  183. {evalview-0.6.1 → evalview-0.7.0}/evalview/core/__init__.py +0 -0
  184. {evalview-0.6.1 → evalview-0.7.0}/evalview/core/budget.py +0 -0
  185. {evalview-0.6.1 → evalview-0.7.0}/evalview/core/contract_diff.py +0 -0
  186. {evalview-0.6.1 → evalview-0.7.0}/evalview/core/dashboard.py +0 -0
  187. {evalview-0.6.1 → evalview-0.7.0}/evalview/core/eval_profiles.py +0 -0
  188. {evalview-0.6.1 → evalview-0.7.0}/evalview/core/golden.py +0 -0
  189. {evalview-0.6.1 → evalview-0.7.0}/evalview/core/healing.py +0 -0
  190. {evalview-0.6.1 → evalview-0.7.0}/evalview/core/judge_cache.py +0 -0
  191. {evalview-0.6.1 → evalview-0.7.0}/evalview/core/llm_configs.py +0 -0
  192. {evalview-0.6.1 → evalview-0.7.0}/evalview/core/llm_provider.py +0 -0
  193. {evalview-0.6.1 → evalview-0.7.0}/evalview/core/loader.py +0 -0
  194. {evalview-0.6.1 → evalview-0.7.0}/evalview/core/mcp_contract.py +0 -0
  195. {evalview-0.6.1 → evalview-0.7.0}/evalview/core/messages.py +0 -0
  196. {evalview-0.6.1 → evalview-0.7.0}/evalview/core/model_runtime_detector.py +0 -0
  197. {evalview-0.6.1 → evalview-0.7.0}/evalview/core/parallel.py +0 -0
  198. {evalview-0.6.1 → evalview-0.7.0}/evalview/core/pricing.py +0 -0
  199. {evalview-0.6.1 → evalview-0.7.0}/evalview/core/project_state.py +0 -0
  200. {evalview-0.6.1 → evalview-0.7.0}/evalview/core/retry.py +0 -0
  201. {evalview-0.6.1 → evalview-0.7.0}/evalview/core/runner.py +0 -0
  202. {evalview-0.6.1 → evalview-0.7.0}/evalview/core/security.py +0 -0
  203. {evalview-0.6.1 → evalview-0.7.0}/evalview/core/semantic_diff.py +0 -0
  204. {evalview-0.6.1 → evalview-0.7.0}/evalview/core/test_quality.py +0 -0
  205. {evalview-0.6.1 → evalview-0.7.0}/evalview/core/tool_categories.py +0 -0
  206. {evalview-0.6.1 → evalview-0.7.0}/evalview/core/tracing.py +0 -0
  207. {evalview-0.6.1 → evalview-0.7.0}/evalview/core/variant_clusterer.py +0 -0
  208. {evalview-0.6.1 → evalview-0.7.0}/evalview/core/watcher.py +0 -0
  209. {evalview-0.6.1 → evalview-0.7.0}/evalview/evaluators/__init__.py +0 -0
  210. {evalview-0.6.1 → evalview-0.7.0}/evalview/evaluators/hallucination_evaluator.py +0 -0
  211. {evalview-0.6.1 → evalview-0.7.0}/evalview/evaluators/latency_evaluator.py +0 -0
  212. {evalview-0.6.1 → evalview-0.7.0}/evalview/evaluators/output_evaluator.py +0 -0
  213. {evalview-0.6.1 → evalview-0.7.0}/evalview/evaluators/pii_evaluator.py +0 -0
  214. {evalview-0.6.1 → evalview-0.7.0}/evalview/evaluators/safety_evaluator.py +0 -0
  215. {evalview-0.6.1 → evalview-0.7.0}/evalview/evaluators/sequence_evaluator.py +0 -0
  216. {evalview-0.6.1 → evalview-0.7.0}/evalview/evaluators/statistical_evaluator.py +0 -0
  217. {evalview-0.6.1 → evalview-0.7.0}/evalview/evaluators/tool_call_evaluator.py +0 -0
  218. {evalview-0.6.1 → evalview-0.7.0}/evalview/expander.py +0 -0
  219. {evalview-0.6.1 → evalview-0.7.0}/evalview/exporters/__init__.py +0 -0
  220. {evalview-0.6.1 → evalview-0.7.0}/evalview/exporters/html_exporter.py +0 -0
  221. {evalview-0.6.1 → evalview-0.7.0}/evalview/importers/__init__.py +0 -0
  222. {evalview-0.6.1 → evalview-0.7.0}/evalview/importers/log_importer.py +0 -0
  223. {evalview-0.6.1 → evalview-0.7.0}/evalview/openclaw.py +0 -0
  224. {evalview-0.6.1 → evalview-0.7.0}/evalview/recorder.py +0 -0
  225. {evalview-0.6.1 → evalview-0.7.0}/evalview/reporters/__init__.py +0 -0
  226. {evalview-0.6.1 → evalview-0.7.0}/evalview/reporters/json_reporter.py +0 -0
  227. {evalview-0.6.1 → evalview-0.7.0}/evalview/reporters/trace_live_reporter.py +0 -0
  228. {evalview-0.6.1 → evalview-0.7.0}/evalview/reporters/trace_reporter.py +0 -0
  229. {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/__init__.py +0 -0
  230. {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/adapters/__init__.py +0 -0
  231. {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/adapters/base.py +0 -0
  232. {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/adapters/claude_code_adapter.py +0 -0
  233. {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/adapters/codex_adapter.py +0 -0
  234. {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/adapters/crewai_adapter.py +0 -0
  235. {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/adapters/custom_adapter.py +0 -0
  236. {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/adapters/langgraph_adapter.py +0 -0
  237. {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/adapters/openai_assistants_adapter.py +0 -0
  238. {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/adapters/openclaw_adapter.py +0 -0
  239. {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/adapters/registry.py +0 -0
  240. {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/agent_runner.py +0 -0
  241. {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/agent_types.py +0 -0
  242. {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/constants.py +0 -0
  243. {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/evaluators/__init__.py +0 -0
  244. {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/evaluators/deterministic.py +0 -0
  245. {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/evaluators/orchestrator.py +0 -0
  246. {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/evaluators/rubric.py +0 -0
  247. {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/parser.py +0 -0
  248. {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/runner.py +0 -0
  249. {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/security_scanner.py +0 -0
  250. {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/test_generator.py +0 -0
  251. {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/test_helpers.py +0 -0
  252. {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/ui_utils.py +0 -0
  253. {evalview-0.6.1 → evalview-0.7.0}/evalview/skills/validator.py +0 -0
  254. {evalview-0.6.1 → evalview-0.7.0}/evalview/storage/__init__.py +0 -0
  255. {evalview-0.6.1 → evalview-0.7.0}/evalview/storage/database.py +0 -0
  256. {evalview-0.6.1 → evalview-0.7.0}/evalview/telemetry/__init__.py +0 -0
  257. {evalview-0.6.1 → evalview-0.7.0}/evalview/telemetry/client.py +0 -0
  258. {evalview-0.6.1 → evalview-0.7.0}/evalview/telemetry/config.py +0 -0
  259. {evalview-0.6.1 → evalview-0.7.0}/evalview/telemetry/decorators.py +0 -0
  260. {evalview-0.6.1 → evalview-0.7.0}/evalview/telemetry/events.py +0 -0
  261. {evalview-0.6.1 → evalview-0.7.0}/evalview/trace_cmd/__init__.py +0 -0
  262. {evalview-0.6.1 → evalview-0.7.0}/evalview/trace_cmd/collector.py +0 -0
  263. {evalview-0.6.1 → evalview-0.7.0}/evalview/trace_cmd/patcher.py +0 -0
  264. {evalview-0.6.1 → evalview-0.7.0}/evalview/trace_cmd/runner.py +0 -0
  265. {evalview-0.6.1 → evalview-0.7.0}/evalview/tracking/__init__.py +0 -0
  266. {evalview-0.6.1 → evalview-0.7.0}/evalview/tracking/database.py +0 -0
  267. {evalview-0.6.1 → evalview-0.7.0}/evalview/tracking/regression.py +0 -0
  268. {evalview-0.6.1 → evalview-0.7.0}/evalview/version_check.py +0 -0
  269. {evalview-0.6.1 → evalview-0.7.0}/evalview/visualization/__init__.py +0 -0
  270. {evalview-0.6.1 → evalview-0.7.0}/evalview.egg-info/dependency_links.txt +0 -0
  271. {evalview-0.6.1 → evalview-0.7.0}/evalview.egg-info/entry_points.txt +0 -0
  272. {evalview-0.6.1 → evalview-0.7.0}/evalview.egg-info/top_level.txt +0 -0
  273. {evalview-0.6.1 → evalview-0.7.0}/setup.cfg +0 -0
  274. {evalview-0.6.1 → evalview-0.7.0}/tests/test_api.py +0 -0
  275. {evalview-0.6.1 → evalview-0.7.0}/tests/test_celebrations.py +0 -0
  276. {evalview-0.6.1 → evalview-0.7.0}/tests/test_ci_generate_comment.py +0 -0
  277. {evalview-0.6.1 → evalview-0.7.0}/tests/test_diff_engine.py +0 -0
  278. {evalview-0.6.1 → evalview-0.7.0}/tests/test_dogfood_e2e.py +0 -0
  279. {evalview-0.6.1 → evalview-0.7.0}/tests/test_drift_tracker.py +0 -0
  280. {evalview-0.6.1 → evalview-0.7.0}/tests/test_e2e_snapshot_check.py +0 -0
  281. {evalview-0.6.1 → evalview-0.7.0}/tests/test_endpoint_detection.py +0 -0
  282. {evalview-0.6.1 → evalview-0.7.0}/tests/test_evaluator_orchestrator.py +0 -0
  283. {evalview-0.6.1 → evalview-0.7.0}/tests/test_evaluators.py +0 -0
  284. {evalview-0.6.1 → evalview-0.7.0}/tests/test_forbidden_tools.py +0 -0
  285. {evalview-0.6.1 → evalview-0.7.0}/tests/test_generate_cmd.py +0 -0
  286. {evalview-0.6.1 → evalview-0.7.0}/tests/test_golden_store.py +0 -0
  287. {evalview-0.6.1 → evalview-0.7.0}/tests/test_healing.py +0 -0
  288. {evalview-0.6.1 → evalview-0.7.0}/tests/test_init_cmd.py +0 -0
  289. {evalview-0.6.1 → evalview-0.7.0}/tests/test_judge_cache.py +0 -0
  290. {evalview-0.6.1 → evalview-0.7.0}/tests/test_llm_provider.py +0 -0
  291. {evalview-0.6.1 → evalview-0.7.0}/tests/test_loader.py +0 -0
  292. {evalview-0.6.1 → evalview-0.7.0}/tests/test_main_evaluator.py +0 -0
  293. {evalview-0.6.1 → evalview-0.7.0}/tests/test_mcp_contracts.py +0 -0
  294. {evalview-0.6.1 → evalview-0.7.0}/tests/test_mcp_server.py +0 -0
  295. {evalview-0.6.1 → evalview-0.7.0}/tests/test_messages.py +0 -0
  296. {evalview-0.6.1 → evalview-0.7.0}/tests/test_model_runtime_detector.py +0 -0
  297. {evalview-0.6.1 → evalview-0.7.0}/tests/test_multi_turn_evaluation.py +0 -0
  298. {evalview-0.6.1 → evalview-0.7.0}/tests/test_new_features.py +0 -0
  299. {evalview-0.6.1 → evalview-0.7.0}/tests/test_pii_evaluator.py +0 -0
  300. {evalview-0.6.1 → evalview-0.7.0}/tests/test_project_state.py +0 -0
  301. {evalview-0.6.1 → evalview-0.7.0}/tests/test_pytest_plugin.py +0 -0
  302. {evalview-0.6.1 → evalview-0.7.0}/tests/test_run_guidance.py +0 -0
  303. {evalview-0.6.1 → evalview-0.7.0}/tests/test_security.py +0 -0
  304. {evalview-0.6.1 → evalview-0.7.0}/tests/test_semantic_diff.py +0 -0
  305. {evalview-0.6.1 → evalview-0.7.0}/tests/test_snapshot_generated_workflow.py +0 -0
  306. {evalview-0.6.1 → evalview-0.7.0}/tests/test_statistical_evaluator.py +0 -0
  307. {evalview-0.6.1 → evalview-0.7.0}/tests/test_telemetry_client.py +0 -0
  308. {evalview-0.6.1 → evalview-0.7.0}/tests/test_version_check.py +0 -0
  309. {evalview-0.6.1 → evalview-0.7.0}/tests/test_visualization_generators.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: evalview
3
- Version: 0.6.1
3
+ Version: 0.7.0
4
4
  Summary: Open-source testing and regression detection framework for AI agents. Golden baseline diffing, CI/CD integration, works with LangGraph, CrewAI, OpenAI, Anthropic Claude, HuggingFace, Ollama, and MCP.
5
5
  Author-email: EvalView Team <hidai@evalview.com>
6
6
  License-Expression: Apache-2.0
@@ -35,6 +35,7 @@ Requires-Dist: httpx>=0.26.0
35
35
  Requires-Dist: python-dateutil>=2.8.2
36
36
  Requires-Dist: python-dotenv>=1.0.0
37
37
  Requires-Dist: jinja2>=3.0
38
+ Requires-Dist: jsonschema>=4.0.0
38
39
  Provides-Extra: reports
39
40
  Requires-Dist: plotly>=5.0; extra == "reports"
40
41
  Provides-Extra: watch
@@ -88,9 +89,11 @@ Dynamic: license-file
88
89
 
89
90
  ---
90
91
 
91
- Your agent can still return `200` and be wrong. A model or provider update can change tool choice, skip a clarification, or degrade output quality without changing your code or breaking a health check. **EvalView catches those silent regressions before users do.**
92
+ Your agent can still return `200` and be wrong. A model or provider update can change tool choice, skip a clarification, or degrade output quality without changing your code or breaking a health check. **EvalView catches those silent regressions before users do — and gives you the loop to investigate them, grade the confidence, and broadcast the verdict to your team.**
92
93
 
93
- **Traditional tests tell you if your agent is up. EvalView tells you if it still behaves correctly.** It tracks drift across outputs, tools, model IDs, and runtime fingerprints, so you can tell "the provider changed" from "my system regressed."
94
+ **You don't need frontier-lab resources to run a serious agent regression loop.** EvalView gives solo devs, startups, and small AI teams the same core discipline: snapshot behavior, detect drift, classify changes, and review or heal them safely.
95
+
96
+ **Traditional tests tell you if your agent is up. EvalView tells you if it still behaves correctly.** It tracks drift across outputs, tools, model IDs, and runtime fingerprints with graded confidence — not a binary alarm — so you can tell "the provider changed" from "my system regressed."
94
97
 
95
98
  [![demo.gif](assets/demo.gif)](https://github.com/user-attachments/assets/96d8b5f7-3561-44a1-86a4-270fb0d1d8a6)
96
99
 
@@ -103,6 +106,13 @@ Most eval tools stop at detect and compare. EvalView helps you classify changes,
103
106
  - Separate provider/model drift from real system regressions
104
107
  - Auto-heal flaky failures with retries, review gates, and audit logs
105
108
 
109
+ Built for **frontier-lab rigor, startup-team practicality**:
110
+ - targeted behavior runs instead of giant always-on eval suites
111
+ - deterministic diffs first, LLM judgment where it adds signal
112
+ - faster loops from change -> eval -> review -> ship
113
+
114
+ [How we run EvalView with this operating model →](docs/OPERATING_MODEL.md)
115
+
106
116
  ```
107
117
  ✓ login-flow PASSED
108
118
  ⚠ refund-request TOOLS_CHANGED
@@ -112,6 +122,32 @@ Most eval tools stop at detect and compare. EvalView helps you classify changes,
112
122
  Score: 85 → 55 Output similarity: 35%
113
123
  ```
114
124
 
125
+ The money screen is the one-line **verdict** that lands under every check — a single ship/don't-ship decision derived from the diff, quarantine state, cost delta, and drift confidence:
126
+
127
+ ```
128
+ ─────────────────────────────────────────────
129
+ VERDICT: 🛑 BLOCK RELEASE
130
+ ─────────────────────────────────────────────
131
+
132
+ • 1 regression: billing-dispute
133
+ • 1 test changed behavior: refund-request
134
+ • Cost up 14% vs baseline
135
+
136
+ Likely cause & next actions:
137
+
138
+ 1. Rerun statistically to distinguish flake from real drift
139
+ (high severity, high confidence)
140
+ → evalview check --statistical 5
141
+
142
+ 2. Review tool descriptions for: escalate_to_human
143
+ (high severity, high confidence)
144
+ Tool selection changed — usually a prompt edit nudged the model
145
+ → evalview replay refund-request --trace
146
+ → evalview golden update refund-request # if the new path is correct
147
+ ```
148
+
149
+ Four tiers: `SAFE_TO_SHIP`, `SHIP_WITH_QUARANTINE`, `INVESTIGATE`, `BLOCK_RELEASE`. The verdict is part of `--json` output, the PR comment, and the cloud ship page — CLI, CI, and dashboard all tell the same story.
150
+
115
151
  ## Quick Start
116
152
 
117
153
  ```bash
@@ -126,6 +162,28 @@ evalview check # Catch regressions after every change
126
162
 
127
163
  That's it. Three commands to regression-test any AI agent. `init` auto-detects your agent type (chat, tool-use, multi-step, RAG, coding) and configures the right evaluators, thresholds, and assertions.
128
164
 
165
+ **After `check`, the investigative loop:**
166
+
167
+ ```bash
168
+ evalview since # 2-second brief: what's changed since your last run
169
+ evalview progress --since yesterday # delta report: what improved/regressed
170
+ evalview drift # per-test sparklines + incident markers
171
+ evalview slack-digest --webhook $SLACK # post the daily verdict to your team
172
+ ```
173
+
174
+ These four commands turn a red ✗ into an answer — *is it real drift, a known flake, or a provider update?* — before anyone opens Slack. `since` is the habit anchor (daily open-the-terminal glance); `progress` answers "did my changes help?" with a worth-a-commit gate; `drift` visualizes the trend; `slack-digest` broadcasts the verdict. See [Daily Workflow →](#daily-workflow).
175
+
176
+ ### Catch silent drift in closed models
177
+
178
+ Worried that `claude-opus-4-5` might behave differently next week without warning? [`evalview model-check`](#model-drift-detection) runs a zero-judge canary suite directly against the provider and tells you exactly when the model has drifted — no agent required, no calibration.
179
+
180
+ ```bash
181
+ evalview model-check --model claude-opus-4-5-20251101 # first run saves baseline
182
+ evalview model-check --model claude-opus-4-5-20251101 # next week, detects any change
183
+ ```
184
+
185
+ [See the full model-check section →](#model-drift-detection)
186
+
129
187
  <details>
130
188
  <summary><strong>Other install methods</strong></summary>
131
189
 
@@ -165,6 +223,111 @@ evalview init --profile rag # Override auto-detect
165
223
 
166
224
  </details>
167
225
 
226
+ ## Daily Workflow
227
+
228
+ Detection is only the first step. EvalView gives you the full investigative loop — so when a test goes red, you can answer *"is it real drift, a known flake, or a provider update?"* in four commands, before anyone opens Slack.
229
+
230
+ **Open the terminal — `evalview since`**
231
+
232
+ ```
233
+ ╭──────────────────────────── Since your last check ─────────────────────────────╮
234
+ │ │
235
+ │ 95% pass rate across 14 runs │
236
+ │ ⚠ 2 soft change(s) │
237
+ │ ✨ improved: search_cases, summarize_thread │
238
+ │ │
239
+ ╰────────────────────────────────────────────────────────────────────────────────╯
240
+
241
+ Drift sparklines (most-declining first)
242
+ ▇▆▅▄▃▂▁▂ billing-dispute
243
+
244
+ 🔥 Streak: 6 days of clean merges
245
+
246
+ One thing to look at first:
247
+ → evalview replay billing-dispute
248
+ ```
249
+
250
+ `evalview since` is the **2-second habit brief** — one hero number, one concern, one action. It reuses Week 1's fingerprinted history, so the "since your last check" window is accurate whether you ran it 10 minutes ago or 4 days ago. Night owls at 2am and daily shippers at 9am see the same command; the label adapts. It's the command that goes in your `.zshrc` so `evalview since` fires when you open the terminal, before the espresso machine is warm.
251
+
252
+ **Morning — `evalview progress --since yesterday`**
253
+
254
+ ```
255
+ ✨ 3 test(s) now passing that weren't
256
+ ⚠ 1 test(s) regressed
257
+
258
+ Improved:
259
+ + refund-flow
260
+ + order-lookup (at a4f2e91)
261
+
262
+ Regressed:
263
+ − billing-dispute
264
+
265
+ Output similarity: 85.20% → 87.50% ↑ +2.30%
266
+
267
+ Worth a commit:
268
+ ✓ refund-flow (high confidence)
269
+ → evalview golden update refund-flow
270
+ ```
271
+
272
+ A **"worth a commit" gate** (3+ consecutive passes) keeps you from celebrating flakes at 2am.
273
+
274
+ **Triage — `evalview drift billing-dispute`**
275
+
276
+ ```
277
+ Test │ Trend │ Samples │ Slope │ First → Last │ Status
278
+ ─────────────────┼──────────────┼─────────┼────────┼──────────────┼────────
279
+ billing-dispute │ ! │ 20 │ -1.5% │ 90% → 80% │ declining
280
+ │ ▇▆▅▄▃▂▁▂▁▂▁ │ │ │ │
281
+
282
+ Most concerning: billing-dispute — slope -1.50% per check over 20 samples
283
+ → evalview replay billing-dispute --trace
284
+ ```
285
+
286
+ Unicode sparklines + OLS slope + **incident markers** (`!`) show *when* the test flipped. Drift is graded `insufficient_history / stable / low / medium / high` — not a binary alarm.
287
+
288
+ **Verdict — `evalview check --statistical 5`**
289
+
290
+ When the verdict layer returns `INVESTIGATE`, a stability-replay recommendation is auto-injected at position #0 of the action list, surviving the severity sort so you never miss it.
291
+
292
+ **Quarantine — `evalview quarantine list`**
293
+
294
+ ```
295
+ Test │ Owner │ Age │ Flaky │ Trend │ Status │ Reason
296
+ ──────────────────┼─────────┼──────┼───────┼───────┼───────────┼──────────────
297
+ race-condition │ @hidai │ 12d │ 4 │ ↘ │ ⏸ active │ race condition
298
+ db-timeout │ @jane │ 45d │ 8 │ ↗ │ ⏰ STALE │ db timeout
299
+
300
+ ⏰ 1 entry stale — review overdue.
301
+ Either fix the underlying flake or remove from quarantine:
302
+ evalview quarantine remove db-timeout
303
+ ```
304
+
305
+ Known-flaky tests **don't block CI** — but staleness tracking, owner tags, and a flaky-count trend glyph keep the list honest. Governance built in, not a dumping ground.
306
+
307
+ **Broadcast — `evalview slack-digest --webhook $SLACK_WEBHOOK`**
308
+
309
+ ```
310
+ 📊 EvalView digest — yesterday
311
+ 🟢 95% pass rate across 47 runs
312
+
313
+ Drift
314
+ ▇▆▅ billing-dispute
315
+
316
+ ⏰ Stale quarantine
317
+ 1 overdue
318
+ • db-timeout — @jane — 45d
319
+
320
+ 🎯 Next: evalview check --fail-on REGRESSION
321
+
322
+ ✓ Digest posted to Slack.
323
+ ```
324
+
325
+ Stdlib-only Block Kit post (zero new deps). Fails soft on bad webhooks. Ends with **one actionable command** your team can copy-paste from the channel.
326
+
327
+ ---
328
+
329
+ **The loop closes:** detection → investigation → graded verdict → quarantine governance → broadcast. You wake up, run `progress`, triage with `drift`, confirm with `check --statistical`, and the team sees the digest before standup. That's the morning ritual — reach for it before the espresso machine warms up.
330
+
168
331
  ## Why EvalView?
169
332
 
170
333
  Use LangSmith for observability. Use Braintrust for scoring. **Use EvalView for regression gating.**
@@ -190,6 +353,9 @@ Use LangSmith for observability. Use Braintrust for scoring. **Use EvalView for
190
353
  | ⚠️ **TOOLS_CHANGED** | Different tools called | Review the diff |
191
354
  | ⚠️ **OUTPUT_CHANGED** | Same tools, output shifted | Review the diff |
192
355
  | ❌ **REGRESSION** | Score dropped significantly | Fix before shipping |
356
+ | 📉 **DRIFTING** | Trend sliding with graded confidence (low/med/high) | Run `evalview drift <test>` |
357
+ | 🔎 **INVESTIGATE** | Verdict layer wants statistical replay | Run `evalview check --statistical 5` |
358
+ | ⏳ **QUARANTINED** | Known-flaky, excluded from CI exit code | Fix underlying flake or remove |
193
359
 
194
360
  ### Model / Runtime Change Detection
195
361
 
@@ -197,7 +363,7 @@ EvalView does more than compare `model_id`.
197
363
 
198
364
  - **Declared model change**: adapter-reported model changed from baseline
199
365
  - **Runtime fingerprint change**: observed model labels in the trace changed, even when the top-level model name is missing
200
- - **Coordinated drift**: multiple tests shift together in the same check run, which often points to a silent provider rollout or runtime change
366
+ - **Coordinated drift**: multiple tests shift together in the same check run, which often points to a silent provider rollout or runtime change — now **graded `low / medium / high`** via `DriftTracker.classify_drift`, not a binary alarm
201
367
 
202
368
  When detected, `evalview check` surfaces a run-level signal with a classification (`declared` or `suspected`), confidence level, and evidence from fingerprints, retries, and affected tests.
203
369
 
@@ -461,9 +627,13 @@ Works with **LangGraph, CrewAI, OpenAI, Claude, Mistral, HuggingFace, Ollama, MC
461
627
 
462
628
  1. **`evalview init`** — detects your running agent, creates a starter test suite
463
629
  2. **`evalview snapshot`** — runs tests, saves traces as baselines
464
- 3. **`evalview check`** — replays tests, diffs against baselines, opens HTML report
465
- 4. **`evalview watch`** — re-runs checks on every file save
466
- 5. **`evalview monitor`** — continuous checks in production with Slack alerts
630
+ 3. **`evalview check`** — replays tests, diffs against baselines, emits the ship/don't-ship verdict, opens HTML report
631
+ 4. **`evalview since`** — 2-second brief: what's changed since your last run (the daily habit anchor)
632
+ 5. **`evalview watch`** — re-runs checks on every file save
633
+ 6. **`evalview monitor`** — continuous checks in production with Slack alerts
634
+ 7. **`evalview progress --since`** — diff any two points in history with a "worth a commit" gate
635
+ 8. **`evalview drift`** — per-test sparklines, OLS slope, and incident markers
636
+ 9. **`evalview slack-digest`** — post the daily verdict to your team channel
467
637
 
468
638
  <details>
469
639
  <summary><strong>Snapshot management</strong></summary>
@@ -488,16 +658,141 @@ evalview monitor # Check every 5 min
488
658
  evalview monitor --dashboard # Live terminal dashboard
489
659
  evalview monitor --slack-webhook https://hooks.slack.com/services/...
490
660
  evalview monitor --history monitor.jsonl # JSONL for dashboards
661
+ evalview monitor --incidents # Log confirmed regressions for `evalview autopr`
491
662
  ```
492
663
 
493
664
  New regressions trigger Slack alerts. Recoveries send all-clear. No spam on persistent failures.
494
665
 
666
+ **Every alert is a promise.** The monitor requires **two consecutive failing cycles** before it pages a human — a single blip self-resolves silently and never interrupts anyone. If a test must alert on the first failure (auth, payments, PII, refund paths), mark it `gate: strict` in its YAML and it bypasses the gate, re-alerting every cycle until it passes.
667
+
668
+ Suppressed failures are never hidden: `evalview slack-digest` renders a Noise section listing every test the gate swallowed, how many times it self-resolved, and a visible false-positive rate (`3 suppressed / 12 fired = 25% noise`). See [`evalview/core/noise_tracker.py`](evalview/core/noise_tracker.py) for the full design — confirmation gate, coordinated-incident collapse, and the `.evalview/noise.jsonl` metric.
669
+
495
670
  [Monitor config options →](docs/CLI_REFERENCE.md)
496
671
 
672
+ ## Auto-PR Regression Tests From Production Incidents
673
+
674
+ `evalview autopr` closes the loop: **production failure → pinned regression test → pull request**, with zero LLM calls and zero manual YAML writing.
675
+
676
+ ```bash
677
+ evalview monitor --incidents # Monitor writes .evalview/incidents.jsonl
678
+ evalview autopr --dry-run # Preview what would be generated
679
+ evalview autopr # Write tests/regressions/*.yaml
680
+ evalview autopr --open-pr # + commit + push + gh pr create
681
+ ```
682
+
683
+ The synthesizer is **pure and deterministic** — no network, no LLM — so it runs instantly in CI. For each confirmed regression in `.evalview/incidents.jsonl` it:
684
+
685
+ - builds a `tests/regressions/<slug>.yaml` that pins the query, the baseline tool sequence (`expected.tools`), the newly-appeared tools (`forbidden_tools`), and short phrases from the bad output (`output.not_contains`)
686
+ - tags the test `suite_type: regression`, `gate: strict`, and stamps the incident metadata into `meta.incident` so `autopr` can skip it on subsequent runs
687
+ - defaults to `min_score: 90` — regression tests are a safety net, not a capability benchmark
688
+
689
+ ```yaml
690
+ # tests/regressions/refund-request-b3c4d5e6.yaml (auto-generated)
691
+ name: regression_refund-request_2026-04-14
692
+ description: Auto-generated from production incident (REGRESSION) at 2026-04-14T12:34:56Z ...
693
+ input:
694
+ query: "I want a refund for order #123"
695
+ expected:
696
+ tools: [lookup_order, check_policy, process_refund]
697
+ forbidden_tools: [escalate_to_human] # appeared only in the failing trace
698
+ output:
699
+ not_contains: ["Sure, I've processed your refund for $999."]
700
+ thresholds:
701
+ min_score: 90.0
702
+ suite_type: regression
703
+ gate: strict
704
+ tags: [incident, autopr]
705
+ ```
706
+
707
+ **Wire it into GitHub Actions:** copy [`examples/github-workflow-autopr.yml`](examples/github-workflow-autopr.yml) to `.github/workflows/evalview-autopr.yml` — the workflow runs `monitor` then `autopr --open-pr --require-new` on a schedule. Every production regression arrives as a reviewable PR, and your hallucination test suite grows by itself.
708
+
709
+ **The CLI is fully local.** `evalview monitor` + `evalview autopr` run entirely on your machine — local files, `gh pr create`, no network calls, no cloud account required. The primitive is free and open-source forever.
710
+
711
+ ## Model Drift Detection
712
+
713
+ Closed models update silently. `evalview model-check` is a dedicated command that runs a fixed structural canary suite directly against the provider and tells you when the model itself has changed — no agent, no LLM judge, no calibration required.
714
+
715
+ ```bash
716
+ # Save a baseline snapshot the day you deploy
717
+ evalview model-check --model claude-opus-4-5-20251101
718
+
719
+ # Run it weekly — detects any behavioral change against that baseline
720
+ evalview model-check --model claude-opus-4-5-20251101
721
+ ```
722
+
723
+ **Example output when drift is detected:**
724
+
725
+ ```
726
+ EvalView model-check
727
+ Model: claude-opus-4-5-20251101
728
+ Provider: anthropic
729
+ Suite: canary v1.public (15 prompts, sha256:6b8e925a5543…)
730
+ Runs/prompt: 1
731
+ Temperature: 0.0
732
+ Cost: $0.0228
733
+
734
+ vs reference (2026-04-10, 7d ago)
735
+ Drift: MODEL (MEDIUM confidence)
736
+ Pass rate: 15/15 → 13/15 (-13.3%)
737
+ Flipped: tool_choice_refund_first_step, json_schema_order_summary
738
+
739
+ vs previous (2026-04-17, 0d ago)
740
+ Drift: NONE
741
+ Pass rate: 13/15 → 13/15 (+0.0%)
742
+ ```
743
+
744
+ **How it works:**
745
+
746
+ | Check type | What it catches |
747
+ |------------|-----------------|
748
+ | **Tool choice** | Did the model pick the right tool, in the right order? |
749
+ | **JSON schema** | Does the output still match the expected structure? |
750
+ | **Refusal** | Did the model refuse when it should (or comply when it should)? |
751
+ | **Exact match** | Does the response match a regex anchor? |
752
+
753
+ Every check runs at `temperature=0` for determinism. Drift is classified as **NONE / WEAK / MEDIUM / STRONG** based on how many prompts flipped pass↔fail. No judge — the signal is structural, not probabilistic.
754
+
755
+ ```bash
756
+ evalview model-check --model claude-opus-4-5-20251101 --dry-run # Cost estimate before running
757
+ evalview model-check --model claude-opus-4-5-20251101 --pin # Pin this run as the new reference
758
+ evalview model-check --model claude-opus-4-5-20251101 --reset-reference # Start a fresh baseline
759
+ evalview model-check --model claude-opus-4-5-20251101 --json # Machine-readable output for CI
760
+ evalview model-check --model claude-opus-4-5-20251101 --suite my-canary.yaml # Bring your own suite
761
+ ```
762
+
763
+ Ships with a **bundled 15-prompt public canary** covering tool selection, JSON schema, refusal behavior, and exact match. Add your own prompts with `--suite`. v1 supports **Anthropic**; OpenAI/Mistral/Cohere land in v1.1.
764
+
765
+ [Full reference → docs/MODEL_CHECK.md](docs/MODEL_CHECK.md)
766
+
767
+ ## Simulation + Decision Rationale
768
+
769
+ Two features that close the gaps the April 2026 agent-eval reports kept calling out: pre-flight what-if testing and structured "why did the agent branch?" logging.
770
+
771
+ **Pre-flight simulation** — `evalview simulate` runs your test suite hermetically against declared mocks (tool calls, LLM responses, HTTP). Deterministic seed, zero cost, works in CI. Declare mocks in the test YAML, fan out with `--variants N`:
772
+
773
+ ```bash
774
+ evalview simulate tests/ --variants 5 --seed 42
775
+ ```
776
+
777
+ **Decision rationale** — every tool_choice / branch gets recorded with the chosen option, alternatives considered, and any model-reported reasoning (Anthropic `thinking` blocks auto-captured). Grouped across runs by `input_hash` so cloud analytics surfaces decision drift before your users notice. Local HTML replay shows it inline.
778
+
779
+ Supported adapters: Anthropic, OpenAI Assistants, LangGraph, CrewAI native.
780
+
781
+ [`docs/SIMULATE.md`](docs/SIMULATE.md) · [`docs/RATIONALE.md`](docs/RATIONALE.md) · [`examples/simulation/`](examples/simulation)
782
+
497
783
  ## Key Features
498
784
 
499
785
  | Feature | Description | Docs |
500
786
  |---------|-------------|------|
787
+ | **Progress command** | `evalview progress --since <date\|sha>` — improved/regressed with "worth a commit" gate | [Above](#daily-workflow) |
788
+ | **Drift command** | `evalview drift` — unicode sparklines, OLS slope, incident markers | [Above](#daily-workflow) |
789
+ | **Slack digest** | `evalview slack-digest` — stdlib Block Kit post with one actionable next-step | [Above](#daily-workflow) |
790
+ | **Flake quarantine** | Known-flaky tests don't block CI; staleness tracking, owner tags, governance | [Above](#daily-workflow) |
791
+ | **Release verdict layer** | Graded drift confidence + auto-injected stability recommendation | [Above](#daily-workflow) |
792
+ | **Recommendation engine** | Suggests the next command from verdict, drift class, and history | [Above](#daily-workflow) |
793
+ | **Model drift detection** | `model-check` — zero-judge canary suite that catches silent model updates | [Docs](#model-drift-detection) |
794
+ | **Simulation harness** | `evalview simulate` — hermetic what-if runs against declared mocks, with `--variants N` fan-out | [Docs](docs/SIMULATE.md) |
795
+ | **Decision rationale** | Structured `tool_choice` / `branch` logging with cross-run grouping for decision-drift detection | [Docs](docs/RATIONALE.md) |
501
796
  | **Assertion wizard** | Analyze captured traffic, suggest smart assertions automatically | [Above](#assertion-wizard--tests-from-real-traffic) |
502
797
  | **Auto-variant discovery** | Run N times, cluster paths, save valid variants | [Above](#auto-variant-discovery--solve-non-determinism) |
503
798
  | **Auto-heal** | Retry flakes, propose variants, escalate structural changes | [Above](#auto-heal--fix-flakes-without-leaving-ci) |
@@ -508,6 +803,7 @@ New regressions trigger Slack alerts. Recoveries send all-clear. No spam on pers
508
803
  | **Multi-reference baselines** | Up to 5 variants for non-deterministic agents | [Docs](docs/GOLDEN_TRACES.md) |
509
804
  | **`forbidden_tools`** | Safety contracts — hard-fail on any violation | [Docs](docs/YAML_SCHEMA.md) |
510
805
  | **Watch mode** | `evalview watch` — re-run checks on file save, with dashboard | [Docs](#watch-mode) |
806
+ | **Model comparison** | `run_eval` / `compare_models` — test one query across N models in parallel | [Docs](#model-comparison) |
511
807
  | **Python API** | `gate()` / `gate_async()` — programmatic regression checks | [Docs](#python-api) |
512
808
  | **PR comments + alerts** | Cost/latency spikes, model changes, collapsible diffs | [Docs](docs/CI_CD.md) |
513
809
  | **Terminal dashboard** | Scorecard, sparkline trends, confidence scoring | — |
@@ -520,6 +816,7 @@ New regressions trigger Slack alerts. Recoveries send all-clear. No spam on pers
520
816
  | **Multi-turn capture** | `capture --multi-turn` records conversations as tests | [Docs](#multi-turn-testing) |
521
817
  | **Semantic similarity** | Embedding-based output comparison | [Docs](docs/EVALUATION_METRICS.md) |
522
818
  | **Production monitoring** | `evalview monitor --dashboard` with Slack alerts and JSONL history | [Docs](#production-monitoring) |
819
+ | **Auto-PR regression tests** | `evalview autopr` turns `.evalview/incidents.jsonl` into pinned regression tests + PRs | [Docs](#auto-pr-regression-tests-from-production-incidents) |
523
820
  | **A/B comparison** | `evalview compare --v1 <url> --v2 <url>` | [Docs](docs/CLI_REFERENCE.md) |
524
821
  | **Test generation** | `evalview generate` — discovers your agent's domain, generates relevant tests | [Docs](docs/TEST_GENERATION.md) |
525
822
  | **Per-turn judge scoring** | Multi-turn output quality scored per turn with conversation context | [Docs](#multi-turn-testing) |
@@ -530,6 +827,7 @@ New regressions trigger Slack alerts. Recoveries send all-clear. No spam on pers
530
827
  | **Verified cost tracking** | Per-test cost breakdown with model pricing rates | [Docs](docs/COST_TRACKING.md) |
531
828
  | **Judge model picker** | Choose GPT, Claude, Gemini, DeepSeek, or Ollama (free) | [Docs](docs/EVALUATION_METRICS.md) |
532
829
  | **Pytest plugin** | `evalview_check` fixture for standard pytest | [Docs](#pytest-plugin) |
830
+ | **Model comparison** | `run_eval` / `compare_models` — parametrize tests across models, auto-detect provider | [Docs](#model-comparison) |
533
831
  | **GitHub Actions job summary** | Results visible in Actions UI, not just PR comments | [Docs](docs/CI_CD.md) |
534
832
  | **Git hooks** | Pre-push regression blocking, zero CI config | [Docs](docs/CI_CD.md) |
535
833
  | **LLM judge caching** | ~80% cost reduction in statistical mode | [Docs](docs/EVALUATION_METRICS.md) |
@@ -618,6 +916,71 @@ pip install evalview # Plugin registers automatically
618
916
  pytest # Runs alongside your existing tests
619
917
  ```
620
918
 
919
+ ## Model Comparison
920
+
921
+ Test the same task across multiple models with one parametrized test. No config files — just a model name and a query.
922
+
923
+ ```python
924
+ import pytest
925
+ import evalview
926
+
927
+ @pytest.mark.parametrize("model", ["claude-opus-4-6", "gpt-4o", "claude-sonnet-4-6"])
928
+ def test_my_task(model):
929
+ result = evalview.run_eval(model, query="Summarize this contract in one sentence.")
930
+ assert evalview.score(result) > 0.8
931
+ ```
932
+
933
+ Provider is auto-detected from the model name. Requires `ANTHROPIC_API_KEY` / `OPENAI_API_KEY` depending on which models you use.
934
+
935
+ **Score against expected output** — token-overlap similarity, no LLM judge needed:
936
+
937
+ ```python
938
+ result = evalview.run_eval(
939
+ "gpt-4o",
940
+ query="What language is Python?",
941
+ expected="Python is a high-level interpreted language.",
942
+ threshold=0.4,
943
+ )
944
+ ```
945
+
946
+ **Custom scorer** — assert specific behavior:
947
+
948
+ ```python
949
+ def has_json(output, expected):
950
+ import json, re
951
+ m = re.search(r"\{.*?\}", output, re.DOTALL)
952
+ try: return 1.0 if json.loads(m.group()) else 0.0
953
+ except: return 0.0
954
+
955
+ result = evalview.run_eval("claude-opus-4-6", query="Return JSON: {name, age}", scorer=has_json)
956
+ assert evalview.score(result) == 1.0
957
+ ```
958
+
959
+ **Run all models in parallel and compare:**
960
+
961
+ ```python
962
+ results = evalview.compare_models(
963
+ query="Explain quantum entanglement in one sentence.",
964
+ models=["claude-opus-4-6", "gpt-4o", "claude-sonnet-4-6"],
965
+ )
966
+ evalview.print_comparison_table(results) # Rich table: score, latency, cost
967
+ best = results[0] # sorted best-first
968
+ ```
969
+
970
+ ```
971
+ ┏━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━┓
972
+ ┃ Model ┃ Score ┃ Latency ┃ Cost ┃ Pass? ┃
973
+ ┡━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━┩
974
+ │ claude-opus-4-6 │ 1.00 │ 842ms │ $0.00312 │ ✓ │
975
+ │ gpt-4o │ 1.00 │ 631ms │ $0.00087 │ ✓ │
976
+ │ claude-sonnet-4-6 │ 1.00 │ 514ms │ $0.00063 │ ✓ │
977
+ └────────────────────┴───────┴──────────┴───────────┴───────┘
978
+ ```
979
+
980
+ `ModelResult` fields: `.model`, `.output`, `.score`, `.latency_ms`, `.cost_usd`, `.passed`, `.error`
981
+
982
+ [Full example →](examples/model_comparison_test.py)
983
+
621
984
  ## Claude Code (MCP)
622
985
 
623
986
  ```bash
@@ -662,6 +1025,7 @@ Then just ask Claude: "did my refactor break anything?" and it runs `run_check`
662
1025
  |---|---|---|
663
1026
  | [Getting Started](docs/GETTING_STARTED.md) | [Golden Traces](docs/GOLDEN_TRACES.md) | [CI/CD](docs/CI_CD.md) |
664
1027
  | [CLI Reference](docs/CLI_REFERENCE.md) | [Evaluation Metrics](docs/EVALUATION_METRICS.md) | [MCP Contracts](docs/MCP_CONTRACTS.md) |
1028
+ | [Simulation](docs/SIMULATE.md) | [Decision Rationale](docs/RATIONALE.md) | |
665
1029
  | [Agent Instructions](AGENT_INSTRUCTIONS.md) | [Agent Recipes](docs/agent-recipes/README.md) | [Ollama Recipe](docs/agent-recipes/integrate-ollama.md) |
666
1030
  | [FAQ](docs/FAQ.md) | [Test Generation](docs/TEST_GENERATION.md) | [Skills Testing](docs/SKILLS_TESTING.md) |
667
1031
  | [YAML Schema](docs/YAML_SCHEMA.md) | [Statistical Mode](docs/STATISTICAL_MODE.md) | [Chat Mode](docs/CHAT_MODE.md) |