evalview 0.6.1__tar.gz → 0.7.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (339) hide show
  1. {evalview-0.6.1/evalview.egg-info → evalview-0.7.1}/PKG-INFO +375 -10
  2. {evalview-0.6.1 → evalview-0.7.1}/README.md +372 -9
  3. evalview-0.7.1/evalview/__init__.py +15 -0
  4. evalview-0.7.1/evalview/adapters/aider_adapter.py +533 -0
  5. {evalview-0.6.1 → evalview-0.7.1}/evalview/adapters/anthropic_adapter.py +33 -0
  6. {evalview-0.6.1 → evalview-0.7.1}/evalview/adapters/crewai_native_adapter.py +35 -3
  7. {evalview-0.6.1 → evalview-0.7.1}/evalview/adapters/langgraph_adapter.py +47 -2
  8. {evalview-0.6.1 → evalview-0.7.1}/evalview/adapters/openai_assistants_adapter.py +18 -0
  9. evalview-0.7.1/evalview/adapters/opencode_adapter.py +387 -0
  10. {evalview-0.6.1 → evalview-0.7.1}/evalview/adapters/pydantic_ai_adapter.py +1 -2
  11. {evalview-0.6.1 → evalview-0.7.1}/evalview/adapters/registry.py +14 -0
  12. {evalview-0.6.1 → evalview-0.7.1}/evalview/api.py +13 -1
  13. evalview-0.7.1/evalview/benchmarks/canary/README.md +93 -0
  14. evalview-0.7.1/evalview/benchmarks/canary/__init__.py +18 -0
  15. evalview-0.7.1/evalview/benchmarks/canary/suite.v1.held-out.yaml +66 -0
  16. evalview-0.7.1/evalview/benchmarks/canary/suite.v1.public.yaml +214 -0
  17. evalview-0.7.1/evalview/chat.py +809 -0
  18. evalview-0.7.1/evalview/chat_commands.py +1010 -0
  19. evalview-0.7.1/evalview/chat_demo.py +562 -0
  20. evalview-0.7.1/evalview/chat_prompt.py +479 -0
  21. {evalview-0.6.1 → evalview-0.7.1}/evalview/chat_runtime.py +41 -13
  22. evalview-0.7.1/evalview/chat_session.py +149 -0
  23. {evalview-0.6.1 → evalview-0.7.1}/evalview/ci/comment.py +255 -2
  24. {evalview-0.6.1 → evalview-0.7.1}/evalview/cli.py +108 -61
  25. evalview-0.7.1/evalview/cloud/push.py +364 -0
  26. evalview-0.7.1/evalview/commands/_check_helpers.py +199 -0
  27. evalview-0.7.1/evalview/commands/_check_verdict.py +373 -0
  28. evalview-0.7.1/evalview/commands/_init_demo_agent.py +178 -0
  29. evalview-0.7.1/evalview/commands/_init_tests.py +280 -0
  30. evalview-0.7.1/evalview/commands/_init_wizard.py +192 -0
  31. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/assertion_wizard.py +2 -7
  32. evalview-0.7.1/evalview/commands/autopr_cmd.py +525 -0
  33. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/badge_cmd.py +2 -3
  34. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/baseline_cmd.py +1 -1
  35. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/benchmark_cmd.py +1 -1
  36. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/check_cmd.py +209 -206
  37. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/check_display.py +319 -20
  38. evalview-0.7.1/evalview/commands/diff_cmd.py +283 -0
  39. evalview-0.7.1/evalview/commands/drift_cmd.py +335 -0
  40. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/generate_cmd.py +1 -1
  41. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/golden_cmd.py +1 -1
  42. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/gym_cmd.py +1 -1
  43. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/hooks_cmd.py +2 -2
  44. evalview-0.7.1/evalview/commands/init_cmd.py +564 -0
  45. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/judge_cmd.py +1 -1
  46. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/listing_cmd.py +52 -16
  47. evalview-0.7.1/evalview/commands/log_cmd.py +212 -0
  48. evalview-0.7.1/evalview/commands/model_check_cmd.py +690 -0
  49. evalview-0.7.1/evalview/commands/model_check_render.py +149 -0
  50. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/monitor_cmd.py +406 -115
  51. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/openclaw_cmd.py +1 -2
  52. evalview-0.7.1/evalview/commands/progress_cmd.py +413 -0
  53. evalview-0.7.1/evalview/commands/quarantine_cmd.py +190 -0
  54. evalview-0.7.1/evalview/commands/replay_trace_cmd.py +256 -0
  55. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/run/_adapters.py +40 -2
  56. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/run/_cmd.py +41 -322
  57. evalview-0.7.1/evalview/commands/run/_helpers.py +360 -0
  58. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/run/_reporter.py +63 -5
  59. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/shared.py +119 -126
  60. evalview-0.7.1/evalview/commands/simulate_cmd.py +213 -0
  61. evalview-0.7.1/evalview/commands/since_cmd.py +536 -0
  62. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/skill_cmd.py +20 -2
  63. evalview-0.7.1/evalview/commands/slack_digest_cmd.py +422 -0
  64. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/snapshot_cmd.py +133 -59
  65. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/telemetry_cmd.py +1 -1
  66. evalview-0.7.1/evalview/commands/validate_cmd.py +209 -0
  67. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/visual_cmd.py +13 -56
  68. evalview-0.7.1/evalview/compare.py +486 -0
  69. {evalview-0.6.1 → evalview-0.7.1}/evalview/core/adapter_factory.py +17 -1
  70. evalview-0.7.1/evalview/core/behavioral_anomalies.py +478 -0
  71. evalview-0.7.1/evalview/core/benchmark_hardening.py +575 -0
  72. evalview-0.7.1/evalview/core/canary_suite.py +171 -0
  73. {evalview-0.6.1 → evalview-0.7.1}/evalview/core/celebrations.py +3 -0
  74. {evalview-0.6.1 → evalview-0.7.1}/evalview/core/config.py +17 -0
  75. {evalview-0.6.1 → evalview-0.7.1}/evalview/core/diff.py +8 -0
  76. evalview-0.7.1/evalview/core/discord_notifier.py +139 -0
  77. evalview-0.7.1/evalview/core/drift_classifier.py +205 -0
  78. evalview-0.7.1/evalview/core/drift_kind.py +49 -0
  79. {evalview-0.6.1 → evalview-0.7.1}/evalview/core/drift_tracker.py +168 -1
  80. {evalview-0.6.1 → evalview-0.7.1}/evalview/core/eval_profiles.py +1 -4
  81. evalview-0.7.1/evalview/core/loader.py +126 -0
  82. evalview-0.7.1/evalview/core/model_check_scoring.py +389 -0
  83. evalview-0.7.1/evalview/core/model_provider_runner.py +224 -0
  84. evalview-0.7.1/evalview/core/model_snapshots.py +321 -0
  85. evalview-0.7.1/evalview/core/noise_tracker.py +529 -0
  86. evalview-0.7.1/evalview/core/observability.py +243 -0
  87. evalview-0.7.1/evalview/core/quarantine.py +450 -0
  88. evalview-0.7.1/evalview/core/rationale.py +238 -0
  89. evalview-0.7.1/evalview/core/recommendations.py +337 -0
  90. evalview-0.7.1/evalview/core/regression_synth.py +271 -0
  91. evalview-0.7.1/evalview/core/replay_pipeline.py +330 -0
  92. {evalview-0.6.1 → evalview-0.7.1}/evalview/core/root_cause.py +243 -8
  93. evalview-0.7.1/evalview/core/simulation.py +336 -0
  94. {evalview-0.6.1 → evalview-0.7.1}/evalview/core/slack_notifier.py +44 -6
  95. evalview-0.7.1/evalview/core/turn_coherence.py +567 -0
  96. {evalview-0.6.1 → evalview-0.7.1}/evalview/core/types.py +253 -1
  97. evalview-0.7.1/evalview/core/verdict.py +242 -0
  98. {evalview-0.6.1 → evalview-0.7.1}/evalview/evaluators/cost_evaluator.py +7 -2
  99. {evalview-0.6.1 → evalview-0.7.1}/evalview/evaluators/evaluator.py +35 -0
  100. {evalview-0.6.1 → evalview-0.7.1}/evalview/evaluators/hallucination_evaluator.py +0 -1
  101. {evalview-0.6.1 → evalview-0.7.1}/evalview/evaluators/output_evaluator.py +0 -1
  102. {evalview-0.6.1 → evalview-0.7.1}/evalview/importers/log_importer.py +121 -6
  103. {evalview-0.6.1 → evalview-0.7.1}/evalview/mcp_server.py +8 -0
  104. {evalview-0.6.1 → evalview-0.7.1}/evalview/pytest_plugin.py +37 -0
  105. evalview-0.7.1/evalview/reporters/_console_coverage.py +182 -0
  106. evalview-0.7.1/evalview/reporters/_console_statistical.py +313 -0
  107. evalview-0.6.1/evalview/reporters/html_reporter.py → evalview-0.7.1/evalview/reporters/_templates.py +201 -544
  108. {evalview-0.6.1 → evalview-0.7.1}/evalview/reporters/console_reporter.py +8 -465
  109. evalview-0.7.1/evalview/reporters/html_reporter.py +637 -0
  110. evalview-0.7.1/evalview/skills/evaluators/_security_checks.py +379 -0
  111. evalview-0.7.1/evalview/skills/evaluators/_security_patterns.py +89 -0
  112. {evalview-0.6.1 → evalview-0.7.1}/evalview/skills/evaluators/deterministic.py +8 -432
  113. {evalview-0.6.1 → evalview-0.7.1}/evalview/skills/types.py +15 -1
  114. evalview-0.7.1/evalview/templates/patterns/code-generation.yaml +31 -0
  115. evalview-0.7.1/evalview/templates/patterns/cost-budget.yaml +20 -0
  116. evalview-0.7.1/evalview/templates/patterns/customer-support.yaml +31 -0
  117. evalview-0.7.1/evalview/templates/patterns/data-analysis.yaml +31 -0
  118. evalview-0.7.1/evalview/templates/patterns/latency-budget.yaml +20 -0
  119. evalview-0.7.1/evalview/templates/patterns/multi-turn-memory.yaml +28 -0
  120. evalview-0.7.1/evalview/templates/patterns/output-format.yaml +23 -0
  121. evalview-0.7.1/evalview/templates/patterns/rag-citation.yaml +29 -0
  122. evalview-0.7.1/evalview/templates/patterns/rag-grounding.yaml +27 -0
  123. evalview-0.7.1/evalview/templates/patterns/research-synthesis.yaml +34 -0
  124. evalview-0.7.1/evalview/templates/patterns/safety-refusal.yaml +28 -0
  125. evalview-0.7.1/evalview/templates/patterns/tool-error-handling.yaml +26 -0
  126. evalview-0.7.1/evalview/templates/patterns/tool-not-called.yaml +22 -0
  127. evalview-0.7.1/evalview/templates/patterns/tool-sequence.yaml +26 -0
  128. evalview-0.7.1/evalview/templates/patterns/wrong-tool-chosen.yaml +23 -0
  129. {evalview-0.6.1 → evalview-0.7.1}/evalview/test_generation.py +35 -862
  130. evalview-0.7.1/evalview/test_generation_constants.py +188 -0
  131. evalview-0.7.1/evalview/test_generation_discovery.py +158 -0
  132. evalview-0.7.1/evalview/test_generation_seeds.py +260 -0
  133. evalview-0.7.1/evalview/test_generation_synthesis.py +391 -0
  134. evalview-0.7.1/evalview/test_generation_types.py +53 -0
  135. evalview-0.7.1/evalview/visualization/_mermaid.py +96 -0
  136. evalview-0.6.1/evalview/visualization/generators.py → evalview-0.7.1/evalview/visualization/_template.py +113 -778
  137. evalview-0.7.1/evalview/visualization/generators.py +893 -0
  138. {evalview-0.6.1 → evalview-0.7.1/evalview.egg-info}/PKG-INFO +375 -10
  139. {evalview-0.6.1 → evalview-0.7.1}/evalview.egg-info/SOURCES.txt +102 -1
  140. {evalview-0.6.1 → evalview-0.7.1}/evalview.egg-info/requires.txt +4 -0
  141. {evalview-0.6.1 → evalview-0.7.1}/pyproject.toml +12 -1
  142. {evalview-0.6.1 → evalview-0.7.1}/tests/test_adapters.py +80 -0
  143. {evalview-0.6.1 → evalview-0.7.1}/tests/test_api.py +2 -8
  144. evalview-0.7.1/tests/test_autopr.py +472 -0
  145. evalview-0.7.1/tests/test_behavioral_anomalies.py +452 -0
  146. evalview-0.7.1/tests/test_benchmark_hardening.py +520 -0
  147. evalview-0.7.1/tests/test_canary_suite.py +171 -0
  148. {evalview-0.6.1 → evalview-0.7.1}/tests/test_check_cmd.py +224 -0
  149. {evalview-0.6.1 → evalview-0.7.1}/tests/test_check_pipeline.py +81 -0
  150. {evalview-0.6.1 → evalview-0.7.1}/tests/test_config_models.py +1 -0
  151. evalview-0.7.1/tests/test_diff_cmd.py +237 -0
  152. {evalview-0.6.1 → evalview-0.7.1}/tests/test_dogfood_e2e.py +5 -0
  153. evalview-0.7.1/tests/test_drift_classifier.py +147 -0
  154. {evalview-0.6.1 → evalview-0.7.1}/tests/test_evaluator_accuracy.py +4 -5
  155. {evalview-0.6.1 → evalview-0.7.1}/tests/test_healing.py +1 -4
  156. {evalview-0.6.1 → evalview-0.7.1}/tests/test_init_cmd.py +18 -0
  157. {evalview-0.6.1 → evalview-0.7.1}/tests/test_loader.py +79 -0
  158. evalview-0.7.1/tests/test_log_importer_csv.py +132 -0
  159. {evalview-0.6.1 → evalview-0.7.1}/tests/test_mcp_server.py +3 -4
  160. evalview-0.7.1/tests/test_model_check_cmd.py +796 -0
  161. evalview-0.7.1/tests/test_model_check_scoring.py +279 -0
  162. evalview-0.7.1/tests/test_model_snapshots.py +272 -0
  163. {evalview-0.6.1 → evalview-0.7.1}/tests/test_monitor.py +176 -1
  164. {evalview-0.6.1 → evalview-0.7.1}/tests/test_multi_turn_evaluation.py +0 -2
  165. evalview-0.7.1/tests/test_noise_tracker.py +448 -0
  166. evalview-0.7.1/tests/test_observability_regressions.py +330 -0
  167. evalview-0.7.1/tests/test_rationale.py +338 -0
  168. evalview-0.7.1/tests/test_replay_pipeline.py +326 -0
  169. {evalview-0.6.1 → evalview-0.7.1}/tests/test_root_cause.py +380 -22
  170. {evalview-0.6.1 → evalview-0.7.1}/tests/test_run_cmd_tags.py +2 -2
  171. {evalview-0.6.1 → evalview-0.7.1}/tests/test_runner.py +8 -2
  172. evalview-0.7.1/tests/test_simulation.py +359 -0
  173. {evalview-0.6.1 → evalview-0.7.1}/tests/test_snapshot_generated_workflow.py +14 -1
  174. evalview-0.7.1/tests/test_snapshot_json_output.py +247 -0
  175. evalview-0.7.1/tests/test_turn_coherence.py +404 -0
  176. {evalview-0.6.1 → evalview-0.7.1}/tests/test_types.py +261 -0
  177. evalview-0.7.1/tests/test_validate_cmd.py +178 -0
  178. evalview-0.7.1/tests/test_verdict.py +228 -0
  179. evalview-0.7.1/tests/test_week1_integration.py +330 -0
  180. evalview-0.7.1/tests/test_week2_quarantine.py +222 -0
  181. evalview-0.7.1/tests/test_week2_review_fixes.py +282 -0
  182. evalview-0.7.1/tests/test_week2_since_and_comment.py +344 -0
  183. evalview-0.7.1/tests/test_week3_progress_and_drift.py +609 -0
  184. evalview-0.6.1/evalview/__init__.py +0 -6
  185. evalview-0.6.1/evalview/chat.py +0 -2824
  186. evalview-0.6.1/evalview/commands/init_cmd.py +0 -1549
  187. evalview-0.6.1/evalview/core/loader.py +0 -64
  188. {evalview-0.6.1 → evalview-0.7.1}/LICENSE +0 -0
  189. {evalview-0.6.1 → evalview-0.7.1}/NOTICE +0 -0
  190. {evalview-0.6.1 → evalview-0.7.1}/evalview/__main__.py +0 -0
  191. {evalview-0.6.1 → evalview-0.7.1}/evalview/adapters/__init__.py +0 -0
  192. {evalview-0.6.1 → evalview-0.7.1}/evalview/adapters/base.py +0 -0
  193. {evalview-0.6.1 → evalview-0.7.1}/evalview/adapters/cohere_adapter.py +0 -0
  194. {evalview-0.6.1 → evalview-0.7.1}/evalview/adapters/crewai_adapter.py +0 -0
  195. {evalview-0.6.1 → evalview-0.7.1}/evalview/adapters/goose_adapter.py +0 -0
  196. {evalview-0.6.1 → evalview-0.7.1}/evalview/adapters/http_adapter.py +0 -0
  197. {evalview-0.6.1 → evalview-0.7.1}/evalview/adapters/huggingface_adapter.py +0 -0
  198. {evalview-0.6.1 → evalview-0.7.1}/evalview/adapters/mcp_adapter.py +0 -0
  199. {evalview-0.6.1 → evalview-0.7.1}/evalview/adapters/mistral_adapter.py +0 -0
  200. {evalview-0.6.1 → evalview-0.7.1}/evalview/adapters/ollama_adapter.py +0 -0
  201. {evalview-0.6.1 → evalview-0.7.1}/evalview/adapters/openclaw_adapter.py +0 -0
  202. {evalview-0.6.1 → evalview-0.7.1}/evalview/adapters/tapescope_adapter.py +0 -0
  203. {evalview-0.6.1 → evalview-0.7.1}/evalview/benchmarks/__init__.py +0 -0
  204. {evalview-0.6.1 → evalview-0.7.1}/evalview/chat_slash.py +0 -0
  205. {evalview-0.6.1 → evalview-0.7.1}/evalview/ci/__init__.py +0 -0
  206. {evalview-0.6.1 → evalview-0.7.1}/evalview/cloud/__init__.py +0 -0
  207. {evalview-0.6.1 → evalview-0.7.1}/evalview/cloud/auth.py +0 -0
  208. {evalview-0.6.1 → evalview-0.7.1}/evalview/cloud/client.py +0 -0
  209. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/__init__.py +0 -0
  210. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/add_cmd.py +0 -0
  211. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/capture_cmd.py +0 -0
  212. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/chat_cmd.py +0 -0
  213. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/ci_cmd.py +0 -0
  214. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/cloud_cmd.py +0 -0
  215. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/demo_cmd.py +0 -0
  216. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/expand_cmd.py +0 -0
  217. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/feedback_cmd.py +0 -0
  218. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/import_cmd.py +0 -0
  219. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/mcp_cmd.py +0 -0
  220. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/run/__init__.py +0 -0
  221. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/run/_executor.py +0 -0
  222. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/run/_runner.py +0 -0
  223. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/skill_agent_runner.py +0 -0
  224. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/traces_cmd.py +0 -0
  225. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/trends_cmd.py +0 -0
  226. {evalview-0.6.1 → evalview-0.7.1}/evalview/commands/watch_cmd.py +0 -0
  227. {evalview-0.6.1 → evalview-0.7.1}/evalview/core/__init__.py +0 -0
  228. {evalview-0.6.1 → evalview-0.7.1}/evalview/core/budget.py +0 -0
  229. {evalview-0.6.1 → evalview-0.7.1}/evalview/core/contract_diff.py +0 -0
  230. {evalview-0.6.1 → evalview-0.7.1}/evalview/core/dashboard.py +0 -0
  231. {evalview-0.6.1 → evalview-0.7.1}/evalview/core/golden.py +0 -0
  232. {evalview-0.6.1 → evalview-0.7.1}/evalview/core/healing.py +0 -0
  233. {evalview-0.6.1 → evalview-0.7.1}/evalview/core/judge_cache.py +0 -0
  234. {evalview-0.6.1 → evalview-0.7.1}/evalview/core/llm_configs.py +0 -0
  235. {evalview-0.6.1 → evalview-0.7.1}/evalview/core/llm_provider.py +0 -0
  236. {evalview-0.6.1 → evalview-0.7.1}/evalview/core/mcp_contract.py +0 -0
  237. {evalview-0.6.1 → evalview-0.7.1}/evalview/core/messages.py +0 -0
  238. {evalview-0.6.1 → evalview-0.7.1}/evalview/core/model_runtime_detector.py +0 -0
  239. {evalview-0.6.1 → evalview-0.7.1}/evalview/core/parallel.py +0 -0
  240. {evalview-0.6.1 → evalview-0.7.1}/evalview/core/pricing.py +0 -0
  241. {evalview-0.6.1 → evalview-0.7.1}/evalview/core/project_state.py +0 -0
  242. {evalview-0.6.1 → evalview-0.7.1}/evalview/core/retry.py +0 -0
  243. {evalview-0.6.1 → evalview-0.7.1}/evalview/core/runner.py +0 -0
  244. {evalview-0.6.1 → evalview-0.7.1}/evalview/core/security.py +0 -0
  245. {evalview-0.6.1 → evalview-0.7.1}/evalview/core/semantic_diff.py +0 -0
  246. {evalview-0.6.1 → evalview-0.7.1}/evalview/core/test_quality.py +0 -0
  247. {evalview-0.6.1 → evalview-0.7.1}/evalview/core/tool_categories.py +0 -0
  248. {evalview-0.6.1 → evalview-0.7.1}/evalview/core/tracing.py +0 -0
  249. {evalview-0.6.1 → evalview-0.7.1}/evalview/core/variant_clusterer.py +0 -0
  250. {evalview-0.6.1 → evalview-0.7.1}/evalview/core/watcher.py +0 -0
  251. {evalview-0.6.1 → evalview-0.7.1}/evalview/evaluators/__init__.py +0 -0
  252. {evalview-0.6.1 → evalview-0.7.1}/evalview/evaluators/latency_evaluator.py +0 -0
  253. {evalview-0.6.1 → evalview-0.7.1}/evalview/evaluators/pii_evaluator.py +0 -0
  254. {evalview-0.6.1 → evalview-0.7.1}/evalview/evaluators/safety_evaluator.py +0 -0
  255. {evalview-0.6.1 → evalview-0.7.1}/evalview/evaluators/sequence_evaluator.py +0 -0
  256. {evalview-0.6.1 → evalview-0.7.1}/evalview/evaluators/statistical_evaluator.py +0 -0
  257. {evalview-0.6.1 → evalview-0.7.1}/evalview/evaluators/tool_call_evaluator.py +0 -0
  258. {evalview-0.6.1 → evalview-0.7.1}/evalview/expander.py +0 -0
  259. {evalview-0.6.1 → evalview-0.7.1}/evalview/exporters/__init__.py +0 -0
  260. {evalview-0.6.1 → evalview-0.7.1}/evalview/exporters/html_exporter.py +0 -0
  261. {evalview-0.6.1 → evalview-0.7.1}/evalview/importers/__init__.py +0 -0
  262. {evalview-0.6.1 → evalview-0.7.1}/evalview/openclaw.py +0 -0
  263. {evalview-0.6.1 → evalview-0.7.1}/evalview/recorder.py +0 -0
  264. {evalview-0.6.1 → evalview-0.7.1}/evalview/reporters/__init__.py +0 -0
  265. {evalview-0.6.1 → evalview-0.7.1}/evalview/reporters/json_reporter.py +0 -0
  266. {evalview-0.6.1 → evalview-0.7.1}/evalview/reporters/trace_live_reporter.py +0 -0
  267. {evalview-0.6.1 → evalview-0.7.1}/evalview/reporters/trace_reporter.py +0 -0
  268. {evalview-0.6.1 → evalview-0.7.1}/evalview/skills/__init__.py +0 -0
  269. {evalview-0.6.1 → evalview-0.7.1}/evalview/skills/adapters/__init__.py +0 -0
  270. {evalview-0.6.1 → evalview-0.7.1}/evalview/skills/adapters/base.py +0 -0
  271. {evalview-0.6.1 → evalview-0.7.1}/evalview/skills/adapters/claude_code_adapter.py +0 -0
  272. {evalview-0.6.1 → evalview-0.7.1}/evalview/skills/adapters/codex_adapter.py +0 -0
  273. {evalview-0.6.1 → evalview-0.7.1}/evalview/skills/adapters/crewai_adapter.py +0 -0
  274. {evalview-0.6.1 → evalview-0.7.1}/evalview/skills/adapters/custom_adapter.py +0 -0
  275. {evalview-0.6.1 → evalview-0.7.1}/evalview/skills/adapters/langgraph_adapter.py +0 -0
  276. {evalview-0.6.1 → evalview-0.7.1}/evalview/skills/adapters/openai_assistants_adapter.py +0 -0
  277. {evalview-0.6.1 → evalview-0.7.1}/evalview/skills/adapters/openclaw_adapter.py +0 -0
  278. {evalview-0.6.1 → evalview-0.7.1}/evalview/skills/adapters/registry.py +0 -0
  279. {evalview-0.6.1 → evalview-0.7.1}/evalview/skills/agent_runner.py +0 -0
  280. {evalview-0.6.1 → evalview-0.7.1}/evalview/skills/agent_types.py +0 -0
  281. {evalview-0.6.1 → evalview-0.7.1}/evalview/skills/constants.py +0 -0
  282. {evalview-0.6.1 → evalview-0.7.1}/evalview/skills/evaluators/__init__.py +0 -0
  283. {evalview-0.6.1 → evalview-0.7.1}/evalview/skills/evaluators/orchestrator.py +0 -0
  284. {evalview-0.6.1 → evalview-0.7.1}/evalview/skills/evaluators/rubric.py +0 -0
  285. {evalview-0.6.1 → evalview-0.7.1}/evalview/skills/parser.py +0 -0
  286. {evalview-0.6.1 → evalview-0.7.1}/evalview/skills/runner.py +0 -0
  287. {evalview-0.6.1 → evalview-0.7.1}/evalview/skills/security_scanner.py +0 -0
  288. {evalview-0.6.1 → evalview-0.7.1}/evalview/skills/test_generator.py +0 -0
  289. {evalview-0.6.1 → evalview-0.7.1}/evalview/skills/test_helpers.py +0 -0
  290. {evalview-0.6.1 → evalview-0.7.1}/evalview/skills/ui_utils.py +0 -0
  291. {evalview-0.6.1 → evalview-0.7.1}/evalview/skills/validator.py +0 -0
  292. {evalview-0.6.1 → evalview-0.7.1}/evalview/storage/__init__.py +0 -0
  293. {evalview-0.6.1 → evalview-0.7.1}/evalview/storage/database.py +0 -0
  294. {evalview-0.6.1 → evalview-0.7.1}/evalview/telemetry/__init__.py +0 -0
  295. {evalview-0.6.1 → evalview-0.7.1}/evalview/telemetry/client.py +0 -0
  296. {evalview-0.6.1 → evalview-0.7.1}/evalview/telemetry/config.py +0 -0
  297. {evalview-0.6.1 → evalview-0.7.1}/evalview/telemetry/decorators.py +0 -0
  298. {evalview-0.6.1 → evalview-0.7.1}/evalview/telemetry/events.py +0 -0
  299. {evalview-0.6.1 → evalview-0.7.1}/evalview/trace_cmd/__init__.py +0 -0
  300. {evalview-0.6.1 → evalview-0.7.1}/evalview/trace_cmd/collector.py +0 -0
  301. {evalview-0.6.1 → evalview-0.7.1}/evalview/trace_cmd/patcher.py +0 -0
  302. {evalview-0.6.1 → evalview-0.7.1}/evalview/trace_cmd/runner.py +0 -0
  303. {evalview-0.6.1 → evalview-0.7.1}/evalview/tracking/__init__.py +0 -0
  304. {evalview-0.6.1 → evalview-0.7.1}/evalview/tracking/database.py +0 -0
  305. {evalview-0.6.1 → evalview-0.7.1}/evalview/tracking/regression.py +0 -0
  306. {evalview-0.6.1 → evalview-0.7.1}/evalview/version_check.py +0 -0
  307. {evalview-0.6.1 → evalview-0.7.1}/evalview/visualization/__init__.py +0 -0
  308. {evalview-0.6.1 → evalview-0.7.1}/evalview.egg-info/dependency_links.txt +0 -0
  309. {evalview-0.6.1 → evalview-0.7.1}/evalview.egg-info/entry_points.txt +0 -0
  310. {evalview-0.6.1 → evalview-0.7.1}/evalview.egg-info/top_level.txt +0 -0
  311. {evalview-0.6.1 → evalview-0.7.1}/setup.cfg +0 -0
  312. {evalview-0.6.1 → evalview-0.7.1}/tests/test_celebrations.py +0 -0
  313. {evalview-0.6.1 → evalview-0.7.1}/tests/test_ci_generate_comment.py +0 -0
  314. {evalview-0.6.1 → evalview-0.7.1}/tests/test_diff_engine.py +0 -0
  315. {evalview-0.6.1 → evalview-0.7.1}/tests/test_drift_tracker.py +0 -0
  316. {evalview-0.6.1 → evalview-0.7.1}/tests/test_e2e_snapshot_check.py +0 -0
  317. {evalview-0.6.1 → evalview-0.7.1}/tests/test_endpoint_detection.py +0 -0
  318. {evalview-0.6.1 → evalview-0.7.1}/tests/test_evaluator_orchestrator.py +0 -0
  319. {evalview-0.6.1 → evalview-0.7.1}/tests/test_evaluators.py +0 -0
  320. {evalview-0.6.1 → evalview-0.7.1}/tests/test_forbidden_tools.py +0 -0
  321. {evalview-0.6.1 → evalview-0.7.1}/tests/test_generate_cmd.py +0 -0
  322. {evalview-0.6.1 → evalview-0.7.1}/tests/test_golden_store.py +0 -0
  323. {evalview-0.6.1 → evalview-0.7.1}/tests/test_judge_cache.py +0 -0
  324. {evalview-0.6.1 → evalview-0.7.1}/tests/test_llm_provider.py +0 -0
  325. {evalview-0.6.1 → evalview-0.7.1}/tests/test_main_evaluator.py +0 -0
  326. {evalview-0.6.1 → evalview-0.7.1}/tests/test_mcp_contracts.py +0 -0
  327. {evalview-0.6.1 → evalview-0.7.1}/tests/test_messages.py +0 -0
  328. {evalview-0.6.1 → evalview-0.7.1}/tests/test_model_runtime_detector.py +0 -0
  329. {evalview-0.6.1 → evalview-0.7.1}/tests/test_new_features.py +0 -0
  330. {evalview-0.6.1 → evalview-0.7.1}/tests/test_pii_evaluator.py +0 -0
  331. {evalview-0.6.1 → evalview-0.7.1}/tests/test_project_state.py +0 -0
  332. {evalview-0.6.1 → evalview-0.7.1}/tests/test_pytest_plugin.py +0 -0
  333. {evalview-0.6.1 → evalview-0.7.1}/tests/test_run_guidance.py +0 -0
  334. {evalview-0.6.1 → evalview-0.7.1}/tests/test_security.py +0 -0
  335. {evalview-0.6.1 → evalview-0.7.1}/tests/test_semantic_diff.py +0 -0
  336. {evalview-0.6.1 → evalview-0.7.1}/tests/test_statistical_evaluator.py +0 -0
  337. {evalview-0.6.1 → evalview-0.7.1}/tests/test_telemetry_client.py +0 -0
  338. {evalview-0.6.1 → evalview-0.7.1}/tests/test_version_check.py +0 -0
  339. {evalview-0.6.1 → evalview-0.7.1}/tests/test_visualization_generators.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: evalview
3
- Version: 0.6.1
3
+ Version: 0.7.1
4
4
  Summary: Open-source testing and regression detection framework for AI agents. Golden baseline diffing, CI/CD integration, works with LangGraph, CrewAI, OpenAI, Anthropic Claude, HuggingFace, Ollama, and MCP.
5
5
  Author-email: EvalView Team <hidai@evalview.com>
6
6
  License-Expression: Apache-2.0
@@ -35,6 +35,8 @@ Requires-Dist: httpx>=0.26.0
35
35
  Requires-Dist: python-dateutil>=2.8.2
36
36
  Requires-Dist: python-dotenv>=1.0.0
37
37
  Requires-Dist: jinja2>=3.0
38
+ Requires-Dist: jsonschema>=4.0.0
39
+ Requires-Dist: tomli>=2.0; python_version < "3.11"
38
40
  Provides-Extra: reports
39
41
  Requires-Dist: plotly>=5.0; extra == "reports"
40
42
  Provides-Extra: watch
@@ -88,9 +90,11 @@ Dynamic: license-file
88
90
 
89
91
  ---
90
92
 
91
- Your agent can still return `200` and be wrong. A model or provider update can change tool choice, skip a clarification, or degrade output quality without changing your code or breaking a health check. **EvalView catches those silent regressions before users do.**
93
+ Your agent can still return `200` and be wrong. A model or provider update can change tool choice, skip a clarification, or degrade output quality without changing your code or breaking a health check. **EvalView catches those silent regressions before users do — and gives you the loop to investigate them, grade the confidence, and broadcast the verdict to your team.**
92
94
 
93
- **Traditional tests tell you if your agent is up. EvalView tells you if it still behaves correctly.** It tracks drift across outputs, tools, model IDs, and runtime fingerprints, so you can tell "the provider changed" from "my system regressed."
95
+ **You don't need frontier-lab resources to run a serious agent regression loop.** EvalView gives solo devs, startups, and small AI teams the same core discipline: snapshot behavior, detect drift, classify changes, and review or heal them safely.
96
+
97
+ **Traditional tests tell you if your agent is up. EvalView tells you if it still behaves correctly.** It tracks drift across outputs, tools, model IDs, and runtime fingerprints with graded confidence — not a binary alarm — so you can tell "the provider changed" from "my system regressed."
94
98
 
95
99
  [![demo.gif](assets/demo.gif)](https://github.com/user-attachments/assets/96d8b5f7-3561-44a1-86a4-270fb0d1d8a6)
96
100
 
@@ -103,6 +107,13 @@ Most eval tools stop at detect and compare. EvalView helps you classify changes,
103
107
  - Separate provider/model drift from real system regressions
104
108
  - Auto-heal flaky failures with retries, review gates, and audit logs
105
109
 
110
+ Built for **frontier-lab rigor, startup-team practicality**:
111
+ - targeted behavior runs instead of giant always-on eval suites
112
+ - deterministic diffs first, LLM judgment where it adds signal
113
+ - faster loops from change -> eval -> review -> ship
114
+
115
+ [How we run EvalView with this operating model →](docs/OPERATING_MODEL.md)
116
+
106
117
  ```
107
118
  ✓ login-flow PASSED
108
119
  ⚠ refund-request TOOLS_CHANGED
@@ -112,6 +123,32 @@ Most eval tools stop at detect and compare. EvalView helps you classify changes,
112
123
  Score: 85 → 55 Output similarity: 35%
113
124
  ```
114
125
 
126
+ The money screen is the one-line **verdict** that lands under every check — a single ship/don't-ship decision derived from the diff, quarantine state, cost delta, and drift confidence:
127
+
128
+ ```
129
+ ─────────────────────────────────────────────
130
+ VERDICT: 🛑 BLOCK RELEASE
131
+ ─────────────────────────────────────────────
132
+
133
+ • 1 regression: billing-dispute
134
+ • 1 test changed behavior: refund-request
135
+ • Cost up 14% vs baseline
136
+
137
+ Likely cause & next actions:
138
+
139
+ 1. Rerun statistically to distinguish flake from real drift
140
+ (high severity, high confidence)
141
+ → evalview check --statistical 5
142
+
143
+ 2. Review tool descriptions for: escalate_to_human
144
+ (high severity, high confidence)
145
+ Tool selection changed — usually a prompt edit nudged the model
146
+ → evalview replay refund-request --trace
147
+ → evalview golden update refund-request # if the new path is correct
148
+ ```
149
+
150
+ Four tiers: `SAFE_TO_SHIP`, `SHIP_WITH_QUARANTINE`, `INVESTIGATE`, `BLOCK_RELEASE`. The verdict is part of `--json` output, the PR comment, and the cloud ship page — CLI, CI, and dashboard all tell the same story.
151
+
115
152
  ## Quick Start
116
153
 
117
154
  ```bash
@@ -126,11 +163,33 @@ evalview check # Catch regressions after every change
126
163
 
127
164
  That's it. Three commands to regression-test any AI agent. `init` auto-detects your agent type (chat, tool-use, multi-step, RAG, coding) and configures the right evaluators, thresholds, and assertions.
128
165
 
166
+ **After `check`, the investigative loop:**
167
+
168
+ ```bash
169
+ evalview since # 2-second brief: what's changed since your last run
170
+ evalview progress --since yesterday # delta report: what improved/regressed
171
+ evalview drift # per-test sparklines + incident markers
172
+ evalview slack-digest --webhook $SLACK # post the daily verdict to your team
173
+ ```
174
+
175
+ These four commands turn a red ✗ into an answer — *is it real drift, a known flake, or a provider update?* — before anyone opens Slack. `since` is the habit anchor (daily open-the-terminal glance); `progress` answers "did my changes help?" with a worth-a-commit gate; `drift` visualizes the trend; `slack-digest` broadcasts the verdict. See [Daily Workflow →](#daily-workflow).
176
+
177
+ ### Catch silent drift in closed models
178
+
179
+ Worried that `claude-opus-4-5` might behave differently next week without warning? [`evalview model-check`](#model-drift-detection) runs a zero-judge canary suite directly against the provider and tells you exactly when the model has drifted — no agent required, no calibration.
180
+
181
+ ```bash
182
+ evalview model-check --model claude-opus-4-5-20251101 # first run saves baseline
183
+ evalview model-check --model claude-opus-4-5-20251101 # next week, detects any change
184
+ ```
185
+
186
+ [See the full model-check section →](#model-drift-detection)
187
+
129
188
  <details>
130
189
  <summary><strong>Other install methods</strong></summary>
131
190
 
132
191
  ```bash
133
- curl -fsSL https://raw.githubusercontent.com/hidai25/eval-view/main/install.sh | bash
192
+ curl -fsSL https://raw.githubusercontent.com/hidai25/eval-view/main/scripts/install.sh | bash
134
193
  ```
135
194
 
136
195
  </details>
@@ -165,6 +224,111 @@ evalview init --profile rag # Override auto-detect
165
224
 
166
225
  </details>
167
226
 
227
+ ## Daily Workflow
228
+
229
+ Detection is only the first step. EvalView gives you the full investigative loop — so when a test goes red, you can answer *"is it real drift, a known flake, or a provider update?"* in four commands, before anyone opens Slack.
230
+
231
+ **Open the terminal — `evalview since`**
232
+
233
+ ```
234
+ ╭──────────────────────────── Since your last check ─────────────────────────────╮
235
+ │ │
236
+ │ 95% pass rate across 14 runs │
237
+ │ ⚠ 2 soft change(s) │
238
+ │ ✨ improved: search_cases, summarize_thread │
239
+ │ │
240
+ ╰────────────────────────────────────────────────────────────────────────────────╯
241
+
242
+ Drift sparklines (most-declining first)
243
+ ▇▆▅▄▃▂▁▂ billing-dispute
244
+
245
+ 🔥 Streak: 6 days of clean merges
246
+
247
+ One thing to look at first:
248
+ → evalview replay billing-dispute
249
+ ```
250
+
251
+ `evalview since` is the **2-second habit brief** — one hero number, one concern, one action. It reuses Week 1's fingerprinted history, so the "since your last check" window is accurate whether you ran it 10 minutes ago or 4 days ago. Night owls at 2am and daily shippers at 9am see the same command; the label adapts. It's the command that goes in your `.zshrc` so `evalview since` fires when you open the terminal, before the espresso machine is warm.
252
+
253
+ **Morning — `evalview progress --since yesterday`**
254
+
255
+ ```
256
+ ✨ 3 test(s) now passing that weren't
257
+ ⚠ 1 test(s) regressed
258
+
259
+ Improved:
260
+ + refund-flow
261
+ + order-lookup (at a4f2e91)
262
+
263
+ Regressed:
264
+ − billing-dispute
265
+
266
+ Output similarity: 85.20% → 87.50% ↑ +2.30%
267
+
268
+ Worth a commit:
269
+ ✓ refund-flow (high confidence)
270
+ → evalview golden update refund-flow
271
+ ```
272
+
273
+ A **"worth a commit" gate** (3+ consecutive passes) keeps you from celebrating flakes at 2am.
274
+
275
+ **Triage — `evalview drift billing-dispute`**
276
+
277
+ ```
278
+ Test │ Trend │ Samples │ Slope │ First → Last │ Status
279
+ ─────────────────┼──────────────┼─────────┼────────┼──────────────┼────────
280
+ billing-dispute │ ! │ 20 │ -1.5% │ 90% → 80% │ declining
281
+ │ ▇▆▅▄▃▂▁▂▁▂▁ │ │ │ │
282
+
283
+ Most concerning: billing-dispute — slope -1.50% per check over 20 samples
284
+ → evalview replay billing-dispute --trace
285
+ ```
286
+
287
+ Unicode sparklines + OLS slope + **incident markers** (`!`) show *when* the test flipped. Drift is graded `insufficient_history / stable / low / medium / high` — not a binary alarm.
288
+
289
+ **Verdict — `evalview check --statistical 5`**
290
+
291
+ When the verdict layer returns `INVESTIGATE`, a stability-replay recommendation is auto-injected at position #0 of the action list, surviving the severity sort so you never miss it.
292
+
293
+ **Quarantine — `evalview quarantine list`**
294
+
295
+ ```
296
+ Test │ Owner │ Age │ Flaky │ Trend │ Status │ Reason
297
+ ──────────────────┼─────────┼──────┼───────┼───────┼───────────┼──────────────
298
+ race-condition │ @hidai │ 12d │ 4 │ ↘ │ ⏸ active │ race condition
299
+ db-timeout │ @jane │ 45d │ 8 │ ↗ │ ⏰ STALE │ db timeout
300
+
301
+ ⏰ 1 entry stale — review overdue.
302
+ Either fix the underlying flake or remove from quarantine:
303
+ evalview quarantine remove db-timeout
304
+ ```
305
+
306
+ Known-flaky tests **don't block CI** — but staleness tracking, owner tags, and a flaky-count trend glyph keep the list honest. Governance built in, not a dumping ground.
307
+
308
+ **Broadcast — `evalview slack-digest --webhook $SLACK_WEBHOOK`**
309
+
310
+ ```
311
+ 📊 EvalView digest — yesterday
312
+ 🟢 95% pass rate across 47 runs
313
+
314
+ Drift
315
+ ▇▆▅ billing-dispute
316
+
317
+ ⏰ Stale quarantine
318
+ 1 overdue
319
+ • db-timeout — @jane — 45d
320
+
321
+ 🎯 Next: evalview check --fail-on REGRESSION
322
+
323
+ ✓ Digest posted to Slack.
324
+ ```
325
+
326
+ Stdlib-only Block Kit post (zero new deps). Fails soft on bad webhooks. Ends with **one actionable command** your team can copy-paste from the channel.
327
+
328
+ ---
329
+
330
+ **The loop closes:** detection → investigation → graded verdict → quarantine governance → broadcast. You wake up, run `progress`, triage with `drift`, confirm with `check --statistical`, and the team sees the digest before standup. That's the morning ritual — reach for it before the espresso machine warms up.
331
+
168
332
  ## Why EvalView?
169
333
 
170
334
  Use LangSmith for observability. Use Braintrust for scoring. **Use EvalView for regression gating.**
@@ -190,6 +354,9 @@ Use LangSmith for observability. Use Braintrust for scoring. **Use EvalView for
190
354
  | ⚠️ **TOOLS_CHANGED** | Different tools called | Review the diff |
191
355
  | ⚠️ **OUTPUT_CHANGED** | Same tools, output shifted | Review the diff |
192
356
  | ❌ **REGRESSION** | Score dropped significantly | Fix before shipping |
357
+ | 📉 **DRIFTING** | Trend sliding with graded confidence (low/med/high) | Run `evalview drift <test>` |
358
+ | 🔎 **INVESTIGATE** | Verdict layer wants statistical replay | Run `evalview check --statistical 5` |
359
+ | ⏳ **QUARANTINED** | Known-flaky, excluded from CI exit code | Fix underlying flake or remove |
193
360
 
194
361
  ### Model / Runtime Change Detection
195
362
 
@@ -197,7 +364,7 @@ EvalView does more than compare `model_id`.
197
364
 
198
365
  - **Declared model change**: adapter-reported model changed from baseline
199
366
  - **Runtime fingerprint change**: observed model labels in the trace changed, even when the top-level model name is missing
200
- - **Coordinated drift**: multiple tests shift together in the same check run, which often points to a silent provider rollout or runtime change
367
+ - **Coordinated drift**: multiple tests shift together in the same check run, which often points to a silent provider rollout or runtime change — now **graded `low / medium / high`** via `DriftTracker.classify_drift`, not a binary alarm
201
368
 
202
369
  When detected, `evalview check` surfaces a run-level signal with a classification (`declared` or `suspected`), confidence level, and evidence from fingerprints, retries, and affected tests.
203
370
 
@@ -461,9 +628,13 @@ Works with **LangGraph, CrewAI, OpenAI, Claude, Mistral, HuggingFace, Ollama, MC
461
628
 
462
629
  1. **`evalview init`** — detects your running agent, creates a starter test suite
463
630
  2. **`evalview snapshot`** — runs tests, saves traces as baselines
464
- 3. **`evalview check`** — replays tests, diffs against baselines, opens HTML report
465
- 4. **`evalview watch`** — re-runs checks on every file save
466
- 5. **`evalview monitor`** — continuous checks in production with Slack alerts
631
+ 3. **`evalview check`** — replays tests, diffs against baselines, emits the ship/don't-ship verdict, opens HTML report
632
+ 4. **`evalview since`** — 2-second brief: what's changed since your last run (the daily habit anchor)
633
+ 5. **`evalview watch`** — re-runs checks on every file save
634
+ 6. **`evalview monitor`** — continuous checks in production with Slack alerts
635
+ 7. **`evalview progress --since`** — diff any two points in history with a "worth a commit" gate
636
+ 8. **`evalview drift`** — per-test sparklines, OLS slope, and incident markers
637
+ 9. **`evalview slack-digest`** — post the daily verdict to your team channel
467
638
 
468
639
  <details>
469
640
  <summary><strong>Snapshot management</strong></summary>
@@ -488,16 +659,141 @@ evalview monitor # Check every 5 min
488
659
  evalview monitor --dashboard # Live terminal dashboard
489
660
  evalview monitor --slack-webhook https://hooks.slack.com/services/...
490
661
  evalview monitor --history monitor.jsonl # JSONL for dashboards
662
+ evalview monitor --incidents # Log confirmed regressions for `evalview autopr`
491
663
  ```
492
664
 
493
665
  New regressions trigger Slack alerts. Recoveries send all-clear. No spam on persistent failures.
494
666
 
667
+ **Every alert is a promise.** The monitor requires **two consecutive failing cycles** before it pages a human — a single blip self-resolves silently and never interrupts anyone. If a test must alert on the first failure (auth, payments, PII, refund paths), mark it `gate: strict` in its YAML and it bypasses the gate, re-alerting every cycle until it passes.
668
+
669
+ Suppressed failures are never hidden: `evalview slack-digest` renders a Noise section listing every test the gate swallowed, how many times it self-resolved, and a visible false-positive rate (`3 suppressed / 12 fired = 25% noise`). See [`evalview/core/noise_tracker.py`](evalview/core/noise_tracker.py) for the full design — confirmation gate, coordinated-incident collapse, and the `.evalview/noise.jsonl` metric.
670
+
495
671
  [Monitor config options →](docs/CLI_REFERENCE.md)
496
672
 
673
+ ## Auto-PR Regression Tests From Production Incidents
674
+
675
+ `evalview autopr` closes the loop: **production failure → pinned regression test → pull request**, with zero LLM calls and zero manual YAML writing.
676
+
677
+ ```bash
678
+ evalview monitor --incidents # Monitor writes .evalview/incidents.jsonl
679
+ evalview autopr --dry-run # Preview what would be generated
680
+ evalview autopr # Write tests/regressions/*.yaml
681
+ evalview autopr --open-pr # + commit + push + gh pr create
682
+ ```
683
+
684
+ The synthesizer is **pure and deterministic** — no network, no LLM — so it runs instantly in CI. For each confirmed regression in `.evalview/incidents.jsonl` it:
685
+
686
+ - builds a `tests/regressions/<slug>.yaml` that pins the query, the baseline tool sequence (`expected.tools`), the newly-appeared tools (`forbidden_tools`), and short phrases from the bad output (`output.not_contains`)
687
+ - tags the test `suite_type: regression`, `gate: strict`, and stamps the incident metadata into `meta.incident` so `autopr` can skip it on subsequent runs
688
+ - defaults to `min_score: 90` — regression tests are a safety net, not a capability benchmark
689
+
690
+ ```yaml
691
+ # tests/regressions/refund-request-b3c4d5e6.yaml (auto-generated)
692
+ name: regression_refund-request_2026-04-14
693
+ description: Auto-generated from production incident (REGRESSION) at 2026-04-14T12:34:56Z ...
694
+ input:
695
+ query: "I want a refund for order #123"
696
+ expected:
697
+ tools: [lookup_order, check_policy, process_refund]
698
+ forbidden_tools: [escalate_to_human] # appeared only in the failing trace
699
+ output:
700
+ not_contains: ["Sure, I've processed your refund for $999."]
701
+ thresholds:
702
+ min_score: 90.0
703
+ suite_type: regression
704
+ gate: strict
705
+ tags: [incident, autopr]
706
+ ```
707
+
708
+ **Wire it into GitHub Actions:** copy [`examples/github-workflow-autopr.yml`](examples/github-workflow-autopr.yml) to `.github/workflows/evalview-autopr.yml` — the workflow runs `monitor` then `autopr --open-pr --require-new` on a schedule. Every production regression arrives as a reviewable PR, and your hallucination test suite grows by itself.
709
+
710
+ **The CLI is fully local.** `evalview monitor` + `evalview autopr` run entirely on your machine — local files, `gh pr create`, no network calls, no cloud account required. The primitive is free and open-source forever.
711
+
712
+ ## Model Drift Detection
713
+
714
+ Closed models update silently. `evalview model-check` is a dedicated command that runs a fixed structural canary suite directly against the provider and tells you when the model itself has changed — no agent, no LLM judge, no calibration required.
715
+
716
+ ```bash
717
+ # Save a baseline snapshot the day you deploy
718
+ evalview model-check --model claude-opus-4-5-20251101
719
+
720
+ # Run it weekly — detects any behavioral change against that baseline
721
+ evalview model-check --model claude-opus-4-5-20251101
722
+ ```
723
+
724
+ **Example output when drift is detected:**
725
+
726
+ ```
727
+ EvalView model-check
728
+ Model: claude-opus-4-5-20251101
729
+ Provider: anthropic
730
+ Suite: canary v1.public (15 prompts, sha256:6b8e925a5543…)
731
+ Runs/prompt: 1
732
+ Temperature: 0.0
733
+ Cost: $0.0228
734
+
735
+ vs reference (2026-04-10, 7d ago)
736
+ Drift: MODEL (MEDIUM confidence)
737
+ Pass rate: 15/15 → 13/15 (-13.3%)
738
+ Flipped: tool_choice_refund_first_step, json_schema_order_summary
739
+
740
+ vs previous (2026-04-17, 0d ago)
741
+ Drift: NONE
742
+ Pass rate: 13/15 → 13/15 (+0.0%)
743
+ ```
744
+
745
+ **How it works:**
746
+
747
+ | Check type | What it catches |
748
+ |------------|-----------------|
749
+ | **Tool choice** | Did the model pick the right tool, in the right order? |
750
+ | **JSON schema** | Does the output still match the expected structure? |
751
+ | **Refusal** | Did the model refuse when it should (or comply when it should)? |
752
+ | **Exact match** | Does the response match a regex anchor? |
753
+
754
+ Every check runs at `temperature=0` for determinism. Drift is classified as **NONE / WEAK / MEDIUM / STRONG** based on how many prompts flipped pass↔fail. No judge — the signal is structural, not probabilistic.
755
+
756
+ ```bash
757
+ evalview model-check --model claude-opus-4-5-20251101 --dry-run # Cost estimate before running
758
+ evalview model-check --model claude-opus-4-5-20251101 --pin # Pin this run as the new reference
759
+ evalview model-check --model claude-opus-4-5-20251101 --reset-reference # Start a fresh baseline
760
+ evalview model-check --model claude-opus-4-5-20251101 --json # Machine-readable output for CI
761
+ evalview model-check --model claude-opus-4-5-20251101 --suite my-canary.yaml # Bring your own suite
762
+ ```
763
+
764
+ Ships with a **bundled 15-prompt public canary** covering tool selection, JSON schema, refusal behavior, and exact match. Add your own prompts with `--suite`. v1 supports **Anthropic**; OpenAI/Mistral/Cohere land in v1.1.
765
+
766
+ [Full reference → docs/MODEL_CHECK.md](docs/MODEL_CHECK.md)
767
+
768
+ ## Simulation + Decision Rationale
769
+
770
+ Two features that close the gaps the April 2026 agent-eval reports kept calling out: pre-flight what-if testing and structured "why did the agent branch?" logging.
771
+
772
+ **Pre-flight simulation** — `evalview simulate` runs your test suite hermetically against declared mocks (tool calls, LLM responses, HTTP). Deterministic seed, zero cost, works in CI. Declare mocks in the test YAML, fan out with `--variants N`:
773
+
774
+ ```bash
775
+ evalview simulate tests/ --variants 5 --seed 42
776
+ ```
777
+
778
+ **Decision rationale** — every tool_choice / branch gets recorded with the chosen option, alternatives considered, and any model-reported reasoning (Anthropic `thinking` blocks auto-captured). Grouped across runs by `input_hash` so cloud analytics surfaces decision drift before your users notice. Local HTML replay shows it inline.
779
+
780
+ Supported adapters: Anthropic, OpenAI Assistants, LangGraph, CrewAI native.
781
+
782
+ [`docs/SIMULATE.md`](docs/SIMULATE.md) · [`docs/RATIONALE.md`](docs/RATIONALE.md) · [`examples/simulation/`](examples/simulation)
783
+
497
784
  ## Key Features
498
785
 
499
786
  | Feature | Description | Docs |
500
787
  |---------|-------------|------|
788
+ | **Progress command** | `evalview progress --since <date\|sha>` — improved/regressed with "worth a commit" gate | [Above](#daily-workflow) |
789
+ | **Drift command** | `evalview drift` — unicode sparklines, OLS slope, incident markers | [Above](#daily-workflow) |
790
+ | **Slack digest** | `evalview slack-digest` — stdlib Block Kit post with one actionable next-step | [Above](#daily-workflow) |
791
+ | **Flake quarantine** | Known-flaky tests don't block CI; staleness tracking, owner tags, governance | [Above](#daily-workflow) |
792
+ | **Release verdict layer** | Graded drift confidence + auto-injected stability recommendation | [Above](#daily-workflow) |
793
+ | **Recommendation engine** | Suggests the next command from verdict, drift class, and history | [Above](#daily-workflow) |
794
+ | **Model drift detection** | `model-check` — zero-judge canary suite that catches silent model updates | [Docs](#model-drift-detection) |
795
+ | **Simulation harness** | `evalview simulate` — hermetic what-if runs against declared mocks, with `--variants N` fan-out | [Docs](docs/SIMULATE.md) |
796
+ | **Decision rationale** | Structured `tool_choice` / `branch` logging with cross-run grouping for decision-drift detection | [Docs](docs/RATIONALE.md) |
501
797
  | **Assertion wizard** | Analyze captured traffic, suggest smart assertions automatically | [Above](#assertion-wizard--tests-from-real-traffic) |
502
798
  | **Auto-variant discovery** | Run N times, cluster paths, save valid variants | [Above](#auto-variant-discovery--solve-non-determinism) |
503
799
  | **Auto-heal** | Retry flakes, propose variants, escalate structural changes | [Above](#auto-heal--fix-flakes-without-leaving-ci) |
@@ -508,6 +804,7 @@ New regressions trigger Slack alerts. Recoveries send all-clear. No spam on pers
508
804
  | **Multi-reference baselines** | Up to 5 variants for non-deterministic agents | [Docs](docs/GOLDEN_TRACES.md) |
509
805
  | **`forbidden_tools`** | Safety contracts — hard-fail on any violation | [Docs](docs/YAML_SCHEMA.md) |
510
806
  | **Watch mode** | `evalview watch` — re-run checks on file save, with dashboard | [Docs](#watch-mode) |
807
+ | **Model comparison** | `run_eval` / `compare_models` — test one query across N models in parallel | [Docs](#model-comparison) |
511
808
  | **Python API** | `gate()` / `gate_async()` — programmatic regression checks | [Docs](#python-api) |
512
809
  | **PR comments + alerts** | Cost/latency spikes, model changes, collapsible diffs | [Docs](docs/CI_CD.md) |
513
810
  | **Terminal dashboard** | Scorecard, sparkline trends, confidence scoring | — |
@@ -520,6 +817,7 @@ New regressions trigger Slack alerts. Recoveries send all-clear. No spam on pers
520
817
  | **Multi-turn capture** | `capture --multi-turn` records conversations as tests | [Docs](#multi-turn-testing) |
521
818
  | **Semantic similarity** | Embedding-based output comparison | [Docs](docs/EVALUATION_METRICS.md) |
522
819
  | **Production monitoring** | `evalview monitor --dashboard` with Slack alerts and JSONL history | [Docs](#production-monitoring) |
820
+ | **Auto-PR regression tests** | `evalview autopr` turns `.evalview/incidents.jsonl` into pinned regression tests + PRs | [Docs](#auto-pr-regression-tests-from-production-incidents) |
523
821
  | **A/B comparison** | `evalview compare --v1 <url> --v2 <url>` | [Docs](docs/CLI_REFERENCE.md) |
524
822
  | **Test generation** | `evalview generate` — discovers your agent's domain, generates relevant tests | [Docs](docs/TEST_GENERATION.md) |
525
823
  | **Per-turn judge scoring** | Multi-turn output quality scored per turn with conversation context | [Docs](#multi-turn-testing) |
@@ -530,6 +828,7 @@ New regressions trigger Slack alerts. Recoveries send all-clear. No spam on pers
530
828
  | **Verified cost tracking** | Per-test cost breakdown with model pricing rates | [Docs](docs/COST_TRACKING.md) |
531
829
  | **Judge model picker** | Choose GPT, Claude, Gemini, DeepSeek, or Ollama (free) | [Docs](docs/EVALUATION_METRICS.md) |
532
830
  | **Pytest plugin** | `evalview_check` fixture for standard pytest | [Docs](#pytest-plugin) |
831
+ | **Model comparison** | `run_eval` / `compare_models` — parametrize tests across models, auto-detect provider | [Docs](#model-comparison) |
533
832
  | **GitHub Actions job summary** | Results visible in Actions UI, not just PR comments | [Docs](docs/CI_CD.md) |
534
833
  | **Git hooks** | Pre-push regression blocking, zero CI config | [Docs](docs/CI_CD.md) |
535
834
  | **LLM judge caching** | ~80% cost reduction in statistical mode | [Docs](docs/EVALUATION_METRICS.md) |
@@ -618,6 +917,71 @@ pip install evalview # Plugin registers automatically
618
917
  pytest # Runs alongside your existing tests
619
918
  ```
620
919
 
920
+ ## Model Comparison
921
+
922
+ Test the same task across multiple models with one parametrized test. No config files — just a model name and a query.
923
+
924
+ ```python
925
+ import pytest
926
+ import evalview
927
+
928
+ @pytest.mark.parametrize("model", ["claude-opus-4-6", "gpt-4o", "claude-sonnet-4-6"])
929
+ def test_my_task(model):
930
+ result = evalview.run_eval(model, query="Summarize this contract in one sentence.")
931
+ assert evalview.score(result) > 0.8
932
+ ```
933
+
934
+ Provider is auto-detected from the model name. Requires `ANTHROPIC_API_KEY` / `OPENAI_API_KEY` depending on which models you use.
935
+
936
+ **Score against expected output** — token-overlap similarity, no LLM judge needed:
937
+
938
+ ```python
939
+ result = evalview.run_eval(
940
+ "gpt-4o",
941
+ query="What language is Python?",
942
+ expected="Python is a high-level interpreted language.",
943
+ threshold=0.4,
944
+ )
945
+ ```
946
+
947
+ **Custom scorer** — assert specific behavior:
948
+
949
+ ```python
950
+ def has_json(output, expected):
951
+ import json, re
952
+ m = re.search(r"\{.*?\}", output, re.DOTALL)
953
+ try: return 1.0 if json.loads(m.group()) else 0.0
954
+ except: return 0.0
955
+
956
+ result = evalview.run_eval("claude-opus-4-6", query="Return JSON: {name, age}", scorer=has_json)
957
+ assert evalview.score(result) == 1.0
958
+ ```
959
+
960
+ **Run all models in parallel and compare:**
961
+
962
+ ```python
963
+ results = evalview.compare_models(
964
+ query="Explain quantum entanglement in one sentence.",
965
+ models=["claude-opus-4-6", "gpt-4o", "claude-sonnet-4-6"],
966
+ )
967
+ evalview.print_comparison_table(results) # Rich table: score, latency, cost
968
+ best = results[0] # sorted best-first
969
+ ```
970
+
971
+ ```
972
+ ┏━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━┓
973
+ ┃ Model ┃ Score ┃ Latency ┃ Cost ┃ Pass? ┃
974
+ ┡━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━┩
975
+ │ claude-opus-4-6 │ 1.00 │ 842ms │ $0.00312 │ ✓ │
976
+ │ gpt-4o │ 1.00 │ 631ms │ $0.00087 │ ✓ │
977
+ │ claude-sonnet-4-6 │ 1.00 │ 514ms │ $0.00063 │ ✓ │
978
+ └────────────────────┴───────┴──────────┴───────────┴───────┘
979
+ ```
980
+
981
+ `ModelResult` fields: `.model`, `.output`, `.score`, `.latency_ms`, `.cost_usd`, `.passed`, `.error`
982
+
983
+ [Full example →](examples/model_comparison_test.py)
984
+
621
985
  ## Claude Code (MCP)
622
986
 
623
987
  ```bash
@@ -648,7 +1012,7 @@ Then just ask Claude: "did my refactor break anything?" and it runs `run_check`
648
1012
 
649
1013
  **Works with your coding agent out of the box.** Ask Cursor, Claude Code, or Copilot to add regression tests, build a new adapter, or debug a failing check — EvalView ships the architecture maps and task recipes they need to get it right on the first try.
650
1014
 
651
- - [AGENT_INSTRUCTIONS.md](AGENT_INSTRUCTIONS.md) — architecture map, contracts, invariants, verification commands
1015
+ - [AGENTS.md](AGENTS.md) — architecture map, contracts, invariants, verification commands
652
1016
  - [Agent Recipes](docs/agent-recipes/README.md) — task-specific playbooks for common extensions
653
1017
  - [Add an Adapter](docs/agent-recipes/add-adapter.md)
654
1018
  - [Add an Evaluator](docs/agent-recipes/add-evaluator.md)
@@ -662,7 +1026,8 @@ Then just ask Claude: "did my refactor break anything?" and it runs `run_check`
662
1026
  |---|---|---|
663
1027
  | [Getting Started](docs/GETTING_STARTED.md) | [Golden Traces](docs/GOLDEN_TRACES.md) | [CI/CD](docs/CI_CD.md) |
664
1028
  | [CLI Reference](docs/CLI_REFERENCE.md) | [Evaluation Metrics](docs/EVALUATION_METRICS.md) | [MCP Contracts](docs/MCP_CONTRACTS.md) |
665
- | [Agent Instructions](AGENT_INSTRUCTIONS.md) | [Agent Recipes](docs/agent-recipes/README.md) | [Ollama Recipe](docs/agent-recipes/integrate-ollama.md) |
1029
+ | [Simulation](docs/SIMULATE.md) | [Decision Rationale](docs/RATIONALE.md) | |
1030
+ | [Agent Instructions](AGENTS.md) | [Agent Recipes](docs/agent-recipes/README.md) | [Ollama Recipe](docs/agent-recipes/integrate-ollama.md) |
666
1031
  | [FAQ](docs/FAQ.md) | [Test Generation](docs/TEST_GENERATION.md) | [Skills Testing](docs/SKILLS_TESTING.md) |
667
1032
  | [YAML Schema](docs/YAML_SCHEMA.md) | [Statistical Mode](docs/STATISTICAL_MODE.md) | [Chat Mode](docs/CHAT_MODE.md) |
668
1033
  | [Framework Support](docs/FRAMEWORK_SUPPORT.md) | [Behavior Coverage](docs/BEHAVIOR_COVERAGE.md) | [Debugging](docs/DEBUGGING.md) |