evalview 0.8.0__tar.gz → 0.9.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (383) hide show
  1. evalview-0.9.0/PKG-INFO +249 -0
  2. evalview-0.9.0/README.md +170 -0
  3. {evalview-0.8.0 → evalview-0.9.0}/evalview/__init__.py +1 -1
  4. {evalview-0.8.0 → evalview-0.9.0}/evalview/adapters/cohere_adapter.py +14 -5
  5. {evalview-0.8.0 → evalview-0.9.0}/evalview/adapters/crewai_native_adapter.py +5 -2
  6. {evalview-0.8.0 → evalview-0.9.0}/evalview/adapters/goose_adapter.py +7 -3
  7. {evalview-0.8.0 → evalview-0.9.0}/evalview/adapters/http_adapter.py +15 -5
  8. {evalview-0.8.0 → evalview-0.9.0}/evalview/adapters/langgraph_adapter.py +3 -2
  9. {evalview-0.8.0 → evalview-0.9.0}/evalview/adapters/mcp_adapter.py +4 -0
  10. {evalview-0.8.0 → evalview-0.9.0}/evalview/adapters/mistral_adapter.py +19 -8
  11. {evalview-0.8.0 → evalview-0.9.0}/evalview/adapters/ollama_adapter.py +1 -1
  12. evalview-0.9.0/evalview/adapters/openai_assistants_adapter.py +473 -0
  13. {evalview-0.8.0 → evalview-0.9.0}/evalview/adapters/openclaw_adapter.py +7 -3
  14. {evalview-0.8.0 → evalview-0.9.0}/evalview/adapters/pydantic_ai_adapter.py +102 -22
  15. {evalview-0.8.0 → evalview-0.9.0}/evalview/adapters/registry.py +7 -0
  16. {evalview-0.8.0 → evalview-0.9.0}/evalview/adapters/tapescope_adapter.py +2 -2
  17. evalview-0.9.0/evalview/adapters/vercel_ai_adapter.py +223 -0
  18. {evalview-0.8.0 → evalview-0.9.0}/evalview/api.py +46 -37
  19. {evalview-0.8.0 → evalview-0.9.0}/evalview/chat.py +1 -1
  20. {evalview-0.8.0 → evalview-0.9.0}/evalview/chat_commands.py +1 -1
  21. {evalview-0.8.0 → evalview-0.9.0}/evalview/chat_prompt.py +37 -30
  22. {evalview-0.8.0 → evalview-0.9.0}/evalview/chat_session.py +43 -27
  23. {evalview-0.8.0 → evalview-0.9.0}/evalview/ci/comment.py +6 -3
  24. evalview-0.9.0/evalview/cloud/auth.py +125 -0
  25. {evalview-0.8.0 → evalview-0.9.0}/evalview/cloud/push.py +66 -16
  26. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/_check_verdict.py +9 -1
  27. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/badge_cmd.py +37 -53
  28. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/check_cmd.py +292 -95
  29. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/check_display.py +69 -5
  30. evalview-0.9.0/evalview/commands/cloud_cmd.py +254 -0
  31. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/drift_cmd.py +2 -2
  32. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/listing_cmd.py +1 -1
  33. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/monitor_cmd.py +57 -26
  34. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/run/_adapters.py +30 -7
  35. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/run/_reporter.py +1 -1
  36. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/shared.py +52 -64
  37. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/snapshot_cmd.py +27 -20
  38. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/trends_cmd.py +16 -1
  39. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/watch_cmd.py +102 -16
  40. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/adapter_factory.py +16 -1
  41. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/chaos.py +3 -3
  42. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/config.py +16 -12
  43. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/diff.py +16 -2
  44. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/drift_tracker.py +44 -37
  45. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/fleet.py +3 -3
  46. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/freshness.py +9 -23
  47. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/goal_drift.py +5 -18
  48. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/golden.py +106 -20
  49. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/healing.py +32 -12
  50. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/loader.py +29 -1
  51. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/noise_tracker.py +7 -3
  52. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/pricing.py +42 -15
  53. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/recommendations.py +9 -0
  54. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/retrieval_lineage.py +5 -16
  55. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/root_cause.py +10 -0
  56. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/runner.py +1 -1
  57. evalview-0.9.0/evalview/core/text.py +23 -0
  58. evalview-0.9.0/evalview/core/trends.py +91 -0
  59. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/types.py +43 -0
  60. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/verdict.py +10 -2
  61. {evalview-0.8.0 → evalview-0.9.0}/evalview/evaluators/evaluator.py +34 -18
  62. {evalview-0.8.0 → evalview-0.9.0}/evalview/evaluators/hallucination_evaluator.py +1 -1
  63. {evalview-0.8.0 → evalview-0.9.0}/evalview/evaluators/output_evaluator.py +15 -21
  64. {evalview-0.8.0 → evalview-0.9.0}/evalview/evaluators/sequence_evaluator.py +5 -5
  65. {evalview-0.8.0 → evalview-0.9.0}/evalview/evaluators/tool_call_evaluator.py +40 -4
  66. {evalview-0.8.0 → evalview-0.9.0}/evalview/mcp_server.py +226 -88
  67. {evalview-0.8.0 → evalview-0.9.0}/evalview/openclaw.py +12 -1
  68. evalview-0.9.0/evalview/py.typed +0 -0
  69. {evalview-0.8.0 → evalview-0.9.0}/evalview/pytest_plugin.py +1 -1
  70. {evalview-0.8.0 → evalview-0.9.0}/evalview/reporters/_console_coverage.py +5 -0
  71. {evalview-0.8.0 → evalview-0.9.0}/evalview/reporters/_console_statistical.py +4 -0
  72. {evalview-0.8.0 → evalview-0.9.0}/evalview/reporters/console_reporter.py +3 -2
  73. {evalview-0.8.0 → evalview-0.9.0}/evalview/reporters/html_reporter.py +9 -8
  74. {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/adapters/__init__.py +1 -1
  75. evalview-0.9.0/evalview/skills/adapters/openai_assistants_adapter.py +531 -0
  76. {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/adapters/registry.py +1 -1
  77. {evalview-0.8.0 → evalview-0.9.0}/evalview/tracking/__init__.py +2 -1
  78. {evalview-0.8.0 → evalview-0.9.0}/evalview/tracking/regression.py +61 -5
  79. {evalview-0.8.0 → evalview-0.9.0}/evalview/visualization/generators.py +10 -2
  80. evalview-0.9.0/evalview.egg-info/PKG-INFO +249 -0
  81. {evalview-0.8.0 → evalview-0.9.0}/evalview.egg-info/SOURCES.txt +23 -1
  82. {evalview-0.8.0 → evalview-0.9.0}/evalview.egg-info/requires.txt +13 -1
  83. {evalview-0.8.0 → evalview-0.9.0}/pyproject.toml +22 -14
  84. evalview-0.9.0/tests/test_action_check_status.py +61 -0
  85. {evalview-0.8.0 → evalview-0.9.0}/tests/test_api.py +7 -4
  86. evalview-0.9.0/tests/test_chat_session.py +84 -0
  87. {evalview-0.8.0 → evalview-0.9.0}/tests/test_check_cmd.py +36 -0
  88. {evalview-0.8.0 → evalview-0.9.0}/tests/test_check_pipeline.py +190 -41
  89. evalview-0.9.0/tests/test_ci_provider_isolation.py +126 -0
  90. evalview-0.9.0/tests/test_cloud_auth_and_push.py +203 -0
  91. evalview-0.9.0/tests/test_documentation_links.py +87 -0
  92. evalview-0.9.0/tests/test_dogfood_ci.py +204 -0
  93. evalview-0.9.0/tests/test_dogfood_harness.py +122 -0
  94. evalview-0.9.0/tests/test_dogfood_issue_routing.py +112 -0
  95. {evalview-0.8.0 → evalview-0.9.0}/tests/test_drift_tracker.py +63 -1
  96. {evalview-0.8.0 → evalview-0.9.0}/tests/test_evaluators.py +163 -3
  97. evalview-0.9.0/tests/test_forbidden_tool_gate.py +140 -0
  98. evalview-0.9.0/tests/test_incomplete_gate.py +210 -0
  99. {evalview-0.8.0 → evalview-0.9.0}/tests/test_loader.py +8 -2
  100. evalview-0.9.0/tests/test_loader_skill_suites.py +111 -0
  101. {evalview-0.8.0 → evalview-0.9.0}/tests/test_mcp_server.py +339 -22
  102. {evalview-0.8.0 → evalview-0.9.0}/tests/test_monitor.py +52 -0
  103. evalview-0.9.0/tests/test_openai_responses_adapter.py +555 -0
  104. evalview-0.9.0/tests/test_pricing.py +74 -0
  105. evalview-0.9.0/tests/test_pydantic_ai_adapter.py +126 -0
  106. evalview-0.9.0/tests/test_regression_tracker.py +279 -0
  107. evalview-0.9.0/tests/test_release_checks.py +167 -0
  108. evalview-0.9.0/tests/test_snapshot_reset.py +202 -0
  109. evalview-0.9.0/tests/test_vercel_ai_adapter.py +357 -0
  110. evalview-0.9.0/tests/test_watch_cmd.py +165 -0
  111. {evalview-0.8.0 → evalview-0.9.0}/tests/test_week3_progress_and_drift.py +2 -0
  112. evalview-0.8.0/PKG-INFO +0 -1063
  113. evalview-0.8.0/README.md +0 -989
  114. evalview-0.8.0/evalview/adapters/openai_assistants_adapter.py +0 -476
  115. evalview-0.8.0/evalview/cloud/auth.py +0 -80
  116. evalview-0.8.0/evalview/cloud/client.py +0 -175
  117. evalview-0.8.0/evalview/commands/cloud_cmd.py +0 -179
  118. evalview-0.8.0/evalview/skills/adapters/openai_assistants_adapter.py +0 -637
  119. evalview-0.8.0/evalview.egg-info/PKG-INFO +0 -1063
  120. {evalview-0.8.0 → evalview-0.9.0}/LICENSE +0 -0
  121. {evalview-0.8.0 → evalview-0.9.0}/NOTICE +0 -0
  122. {evalview-0.8.0 → evalview-0.9.0}/evalview/__main__.py +0 -0
  123. {evalview-0.8.0 → evalview-0.9.0}/evalview/adapters/__init__.py +0 -0
  124. {evalview-0.8.0 → evalview-0.9.0}/evalview/adapters/aider_adapter.py +0 -0
  125. {evalview-0.8.0 → evalview-0.9.0}/evalview/adapters/anthropic_adapter.py +0 -0
  126. {evalview-0.8.0 → evalview-0.9.0}/evalview/adapters/base.py +0 -0
  127. {evalview-0.8.0 → evalview-0.9.0}/evalview/adapters/crewai_adapter.py +0 -0
  128. {evalview-0.8.0 → evalview-0.9.0}/evalview/adapters/huggingface_adapter.py +0 -0
  129. {evalview-0.8.0 → evalview-0.9.0}/evalview/adapters/opencode_adapter.py +0 -0
  130. {evalview-0.8.0 → evalview-0.9.0}/evalview/benchmarks/__init__.py +0 -0
  131. {evalview-0.8.0 → evalview-0.9.0}/evalview/benchmarks/canary/README.md +0 -0
  132. {evalview-0.8.0 → evalview-0.9.0}/evalview/benchmarks/canary/__init__.py +0 -0
  133. {evalview-0.8.0 → evalview-0.9.0}/evalview/benchmarks/canary/suite.v1.held-out.yaml +0 -0
  134. {evalview-0.8.0 → evalview-0.9.0}/evalview/benchmarks/canary/suite.v1.public.yaml +0 -0
  135. {evalview-0.8.0 → evalview-0.9.0}/evalview/chat_demo.py +0 -0
  136. {evalview-0.8.0 → evalview-0.9.0}/evalview/chat_runtime.py +0 -0
  137. {evalview-0.8.0 → evalview-0.9.0}/evalview/chat_slash.py +0 -0
  138. {evalview-0.8.0 → evalview-0.9.0}/evalview/ci/__init__.py +0 -0
  139. {evalview-0.8.0 → evalview-0.9.0}/evalview/cli.py +0 -0
  140. {evalview-0.8.0 → evalview-0.9.0}/evalview/cloud/__init__.py +0 -0
  141. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/__init__.py +0 -0
  142. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/_check_helpers.py +0 -0
  143. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/_init_demo_agent.py +0 -0
  144. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/_init_tests.py +0 -0
  145. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/_init_wizard.py +0 -0
  146. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/add_cmd.py +0 -0
  147. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/assertion_wizard.py +0 -0
  148. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/autopr_cmd.py +0 -0
  149. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/baseline_cmd.py +0 -0
  150. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/benchmark_cmd.py +0 -0
  151. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/capture_cmd.py +0 -0
  152. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/chat_cmd.py +0 -0
  153. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/ci_cmd.py +0 -0
  154. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/demo_cmd.py +0 -0
  155. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/diff_cmd.py +0 -0
  156. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/expand_cmd.py +0 -0
  157. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/feedback_cmd.py +0 -0
  158. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/fleet_cmd.py +0 -0
  159. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/freshness_cmd.py +0 -0
  160. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/generate_cmd.py +0 -0
  161. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/golden_cmd.py +0 -0
  162. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/gym_cmd.py +0 -0
  163. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/hooks_cmd.py +0 -0
  164. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/import_cmd.py +0 -0
  165. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/init_cmd.py +0 -0
  166. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/judge_cmd.py +0 -0
  167. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/log_cmd.py +0 -0
  168. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/mcp_cmd.py +0 -0
  169. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/model_check_cmd.py +0 -0
  170. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/model_check_render.py +0 -0
  171. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/openclaw_cmd.py +0 -0
  172. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/progress_cmd.py +0 -0
  173. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/quarantine_cmd.py +0 -0
  174. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/replay_trace_cmd.py +0 -0
  175. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/run/__init__.py +0 -0
  176. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/run/_cmd.py +0 -0
  177. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/run/_executor.py +0 -0
  178. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/run/_helpers.py +0 -0
  179. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/run/_runner.py +0 -0
  180. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/simulate_cmd.py +0 -0
  181. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/since_cmd.py +0 -0
  182. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/skill_agent_runner.py +0 -0
  183. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/skill_cmd.py +0 -0
  184. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/slack_digest_cmd.py +0 -0
  185. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/telemetry_cmd.py +0 -0
  186. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/traces_cmd.py +0 -0
  187. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/validate_cmd.py +0 -0
  188. {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/visual_cmd.py +0 -0
  189. {evalview-0.8.0 → evalview-0.9.0}/evalview/compare.py +0 -0
  190. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/__init__.py +0 -0
  191. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/behavioral_anomalies.py +0 -0
  192. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/benchmark_hardening.py +0 -0
  193. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/budget.py +0 -0
  194. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/canary_suite.py +0 -0
  195. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/cassette.py +0 -0
  196. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/celebrations.py +0 -0
  197. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/contract_diff.py +0 -0
  198. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/dashboard.py +0 -0
  199. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/discord_notifier.py +0 -0
  200. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/drift_classifier.py +0 -0
  201. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/drift_kind.py +0 -0
  202. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/eval_profiles.py +0 -0
  203. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/judge_cache.py +0 -0
  204. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/llm_configs.py +0 -0
  205. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/llm_provider.py +0 -0
  206. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/mcp_contract.py +0 -0
  207. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/messages.py +0 -0
  208. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/model_check_scoring.py +0 -0
  209. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/model_provider_runner.py +0 -0
  210. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/model_runtime_detector.py +0 -0
  211. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/model_snapshots.py +0 -0
  212. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/observability.py +0 -0
  213. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/otel_semconv.py +0 -0
  214. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/parallel.py +0 -0
  215. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/project_state.py +0 -0
  216. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/quarantine.py +0 -0
  217. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/rationale.py +0 -0
  218. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/regression_synth.py +0 -0
  219. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/replay_pipeline.py +0 -0
  220. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/retry.py +0 -0
  221. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/root_cause_hint.py +0 -0
  222. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/security.py +0 -0
  223. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/semantic_diff.py +0 -0
  224. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/simulation.py +0 -0
  225. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/slack_notifier.py +0 -0
  226. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/test_quality.py +0 -0
  227. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/tool_categories.py +0 -0
  228. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/tracing.py +0 -0
  229. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/turn_coherence.py +0 -0
  230. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/variant_clusterer.py +0 -0
  231. {evalview-0.8.0 → evalview-0.9.0}/evalview/core/watcher.py +0 -0
  232. {evalview-0.8.0 → evalview-0.9.0}/evalview/evaluators/__init__.py +0 -0
  233. {evalview-0.8.0 → evalview-0.9.0}/evalview/evaluators/cost_evaluator.py +0 -0
  234. {evalview-0.8.0 → evalview-0.9.0}/evalview/evaluators/latency_evaluator.py +0 -0
  235. {evalview-0.8.0 → evalview-0.9.0}/evalview/evaluators/pii_evaluator.py +0 -0
  236. {evalview-0.8.0 → evalview-0.9.0}/evalview/evaluators/safety_evaluator.py +0 -0
  237. {evalview-0.8.0 → evalview-0.9.0}/evalview/evaluators/statistical_evaluator.py +0 -0
  238. {evalview-0.8.0 → evalview-0.9.0}/evalview/expander.py +0 -0
  239. {evalview-0.8.0 → evalview-0.9.0}/evalview/exporters/__init__.py +0 -0
  240. {evalview-0.8.0 → evalview-0.9.0}/evalview/exporters/html_exporter.py +0 -0
  241. {evalview-0.8.0 → evalview-0.9.0}/evalview/importers/__init__.py +0 -0
  242. {evalview-0.8.0 → evalview-0.9.0}/evalview/importers/log_importer.py +0 -0
  243. {evalview-0.8.0 → evalview-0.9.0}/evalview/recorder.py +0 -0
  244. {evalview-0.8.0 → evalview-0.9.0}/evalview/reporters/__init__.py +0 -0
  245. {evalview-0.8.0 → evalview-0.9.0}/evalview/reporters/_templates.py +0 -0
  246. {evalview-0.8.0 → evalview-0.9.0}/evalview/reporters/json_reporter.py +0 -0
  247. {evalview-0.8.0 → evalview-0.9.0}/evalview/reporters/trace_live_reporter.py +0 -0
  248. {evalview-0.8.0 → evalview-0.9.0}/evalview/reporters/trace_reporter.py +0 -0
  249. {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/__init__.py +0 -0
  250. {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/adapters/base.py +0 -0
  251. {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/adapters/claude_code_adapter.py +0 -0
  252. {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/adapters/codex_adapter.py +0 -0
  253. {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/adapters/crewai_adapter.py +0 -0
  254. {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/adapters/custom_adapter.py +0 -0
  255. {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/adapters/langgraph_adapter.py +0 -0
  256. {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/adapters/openclaw_adapter.py +0 -0
  257. {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/agent_runner.py +0 -0
  258. {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/agent_types.py +0 -0
  259. {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/constants.py +0 -0
  260. {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/evaluators/__init__.py +0 -0
  261. {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/evaluators/_security_checks.py +0 -0
  262. {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/evaluators/_security_patterns.py +0 -0
  263. {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/evaluators/deterministic.py +0 -0
  264. {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/evaluators/orchestrator.py +0 -0
  265. {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/evaluators/rubric.py +0 -0
  266. {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/parser.py +0 -0
  267. {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/runner.py +0 -0
  268. {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/security_scanner.py +0 -0
  269. {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/test_generator.py +0 -0
  270. {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/test_helpers.py +0 -0
  271. {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/types.py +0 -0
  272. {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/ui_utils.py +0 -0
  273. {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/validator.py +0 -0
  274. {evalview-0.8.0 → evalview-0.9.0}/evalview/storage/__init__.py +0 -0
  275. {evalview-0.8.0 → evalview-0.9.0}/evalview/storage/database.py +0 -0
  276. {evalview-0.8.0 → evalview-0.9.0}/evalview/telemetry/__init__.py +0 -0
  277. {evalview-0.8.0 → evalview-0.9.0}/evalview/telemetry/client.py +0 -0
  278. {evalview-0.8.0 → evalview-0.9.0}/evalview/telemetry/config.py +0 -0
  279. {evalview-0.8.0 → evalview-0.9.0}/evalview/telemetry/decorators.py +0 -0
  280. {evalview-0.8.0 → evalview-0.9.0}/evalview/telemetry/events.py +0 -0
  281. {evalview-0.8.0 → evalview-0.9.0}/evalview/templates/patterns/code-generation.yaml +0 -0
  282. {evalview-0.8.0 → evalview-0.9.0}/evalview/templates/patterns/cost-budget.yaml +0 -0
  283. {evalview-0.8.0 → evalview-0.9.0}/evalview/templates/patterns/customer-support.yaml +0 -0
  284. {evalview-0.8.0 → evalview-0.9.0}/evalview/templates/patterns/data-analysis.yaml +0 -0
  285. {evalview-0.8.0 → evalview-0.9.0}/evalview/templates/patterns/latency-budget.yaml +0 -0
  286. {evalview-0.8.0 → evalview-0.9.0}/evalview/templates/patterns/multi-turn-memory.yaml +0 -0
  287. {evalview-0.8.0 → evalview-0.9.0}/evalview/templates/patterns/output-format.yaml +0 -0
  288. {evalview-0.8.0 → evalview-0.9.0}/evalview/templates/patterns/rag-citation.yaml +0 -0
  289. {evalview-0.8.0 → evalview-0.9.0}/evalview/templates/patterns/rag-grounding.yaml +0 -0
  290. {evalview-0.8.0 → evalview-0.9.0}/evalview/templates/patterns/research-synthesis.yaml +0 -0
  291. {evalview-0.8.0 → evalview-0.9.0}/evalview/templates/patterns/safety-refusal.yaml +0 -0
  292. {evalview-0.8.0 → evalview-0.9.0}/evalview/templates/patterns/tool-error-handling.yaml +0 -0
  293. {evalview-0.8.0 → evalview-0.9.0}/evalview/templates/patterns/tool-not-called.yaml +0 -0
  294. {evalview-0.8.0 → evalview-0.9.0}/evalview/templates/patterns/tool-sequence.yaml +0 -0
  295. {evalview-0.8.0 → evalview-0.9.0}/evalview/templates/patterns/wrong-tool-chosen.yaml +0 -0
  296. {evalview-0.8.0 → evalview-0.9.0}/evalview/test_generation.py +0 -0
  297. {evalview-0.8.0 → evalview-0.9.0}/evalview/test_generation_constants.py +0 -0
  298. {evalview-0.8.0 → evalview-0.9.0}/evalview/test_generation_discovery.py +0 -0
  299. {evalview-0.8.0 → evalview-0.9.0}/evalview/test_generation_seeds.py +0 -0
  300. {evalview-0.8.0 → evalview-0.9.0}/evalview/test_generation_synthesis.py +0 -0
  301. {evalview-0.8.0 → evalview-0.9.0}/evalview/test_generation_types.py +0 -0
  302. {evalview-0.8.0 → evalview-0.9.0}/evalview/trace_cmd/__init__.py +0 -0
  303. {evalview-0.8.0 → evalview-0.9.0}/evalview/trace_cmd/collector.py +0 -0
  304. {evalview-0.8.0 → evalview-0.9.0}/evalview/trace_cmd/patcher.py +0 -0
  305. {evalview-0.8.0 → evalview-0.9.0}/evalview/trace_cmd/runner.py +0 -0
  306. {evalview-0.8.0 → evalview-0.9.0}/evalview/tracking/database.py +0 -0
  307. {evalview-0.8.0 → evalview-0.9.0}/evalview/version_check.py +0 -0
  308. {evalview-0.8.0 → evalview-0.9.0}/evalview/visualization/__init__.py +0 -0
  309. {evalview-0.8.0 → evalview-0.9.0}/evalview/visualization/_mermaid.py +0 -0
  310. {evalview-0.8.0 → evalview-0.9.0}/evalview/visualization/_template.py +0 -0
  311. {evalview-0.8.0 → evalview-0.9.0}/evalview.egg-info/dependency_links.txt +0 -0
  312. {evalview-0.8.0 → evalview-0.9.0}/evalview.egg-info/entry_points.txt +0 -0
  313. {evalview-0.8.0 → evalview-0.9.0}/evalview.egg-info/top_level.txt +0 -0
  314. {evalview-0.8.0 → evalview-0.9.0}/setup.cfg +0 -0
  315. {evalview-0.8.0 → evalview-0.9.0}/tests/test_adapters.py +0 -0
  316. {evalview-0.8.0 → evalview-0.9.0}/tests/test_autopr.py +0 -0
  317. {evalview-0.8.0 → evalview-0.9.0}/tests/test_behavioral_anomalies.py +0 -0
  318. {evalview-0.8.0 → evalview-0.9.0}/tests/test_benchmark_hardening.py +0 -0
  319. {evalview-0.8.0 → evalview-0.9.0}/tests/test_canary_suite.py +0 -0
  320. {evalview-0.8.0 → evalview-0.9.0}/tests/test_cassette.py +0 -0
  321. {evalview-0.8.0 → evalview-0.9.0}/tests/test_celebrations.py +0 -0
  322. {evalview-0.8.0 → evalview-0.9.0}/tests/test_chaos.py +0 -0
  323. {evalview-0.8.0 → evalview-0.9.0}/tests/test_ci_generate_comment.py +0 -0
  324. {evalview-0.8.0 → evalview-0.9.0}/tests/test_config_models.py +0 -0
  325. {evalview-0.8.0 → evalview-0.9.0}/tests/test_diff_cmd.py +0 -0
  326. {evalview-0.8.0 → evalview-0.9.0}/tests/test_diff_engine.py +0 -0
  327. {evalview-0.8.0 → evalview-0.9.0}/tests/test_dogfood_e2e.py +0 -0
  328. {evalview-0.8.0 → evalview-0.9.0}/tests/test_drift_classifier.py +0 -0
  329. {evalview-0.8.0 → evalview-0.9.0}/tests/test_e2e_snapshot_check.py +0 -0
  330. {evalview-0.8.0 → evalview-0.9.0}/tests/test_endpoint_detection.py +0 -0
  331. {evalview-0.8.0 → evalview-0.9.0}/tests/test_evaluator_accuracy.py +0 -0
  332. {evalview-0.8.0 → evalview-0.9.0}/tests/test_evaluator_orchestrator.py +0 -0
  333. {evalview-0.8.0 → evalview-0.9.0}/tests/test_fleet.py +0 -0
  334. {evalview-0.8.0 → evalview-0.9.0}/tests/test_forbidden_tools.py +0 -0
  335. {evalview-0.8.0 → evalview-0.9.0}/tests/test_freshness.py +0 -0
  336. {evalview-0.8.0 → evalview-0.9.0}/tests/test_generate_cmd.py +0 -0
  337. {evalview-0.8.0 → evalview-0.9.0}/tests/test_goal_drift.py +0 -0
  338. {evalview-0.8.0 → evalview-0.9.0}/tests/test_golden_store.py +0 -0
  339. {evalview-0.8.0 → evalview-0.9.0}/tests/test_healing.py +0 -0
  340. {evalview-0.8.0 → evalview-0.9.0}/tests/test_init_cmd.py +0 -0
  341. {evalview-0.8.0 → evalview-0.9.0}/tests/test_judge_cache.py +0 -0
  342. {evalview-0.8.0 → evalview-0.9.0}/tests/test_llm_provider.py +0 -0
  343. {evalview-0.8.0 → evalview-0.9.0}/tests/test_log_importer_csv.py +0 -0
  344. {evalview-0.8.0 → evalview-0.9.0}/tests/test_main_evaluator.py +0 -0
  345. {evalview-0.8.0 → evalview-0.9.0}/tests/test_mcp_contracts.py +0 -0
  346. {evalview-0.8.0 → evalview-0.9.0}/tests/test_messages.py +0 -0
  347. {evalview-0.8.0 → evalview-0.9.0}/tests/test_model_check_cmd.py +0 -0
  348. {evalview-0.8.0 → evalview-0.9.0}/tests/test_model_check_scoring.py +0 -0
  349. {evalview-0.8.0 → evalview-0.9.0}/tests/test_model_runtime_detector.py +0 -0
  350. {evalview-0.8.0 → evalview-0.9.0}/tests/test_model_snapshots.py +0 -0
  351. {evalview-0.8.0 → evalview-0.9.0}/tests/test_multi_turn_evaluation.py +0 -0
  352. {evalview-0.8.0 → evalview-0.9.0}/tests/test_new_features.py +0 -0
  353. {evalview-0.8.0 → evalview-0.9.0}/tests/test_noise_tracker.py +0 -0
  354. {evalview-0.8.0 → evalview-0.9.0}/tests/test_observability_regressions.py +0 -0
  355. {evalview-0.8.0 → evalview-0.9.0}/tests/test_otel_semconv.py +0 -0
  356. {evalview-0.8.0 → evalview-0.9.0}/tests/test_pii_evaluator.py +0 -0
  357. {evalview-0.8.0 → evalview-0.9.0}/tests/test_project_state.py +0 -0
  358. {evalview-0.8.0 → evalview-0.9.0}/tests/test_pytest_plugin.py +0 -0
  359. {evalview-0.8.0 → evalview-0.9.0}/tests/test_rationale.py +0 -0
  360. {evalview-0.8.0 → evalview-0.9.0}/tests/test_replay_pipeline.py +0 -0
  361. {evalview-0.8.0 → evalview-0.9.0}/tests/test_retrieval_lineage.py +0 -0
  362. {evalview-0.8.0 → evalview-0.9.0}/tests/test_root_cause.py +0 -0
  363. {evalview-0.8.0 → evalview-0.9.0}/tests/test_root_cause_hint.py +0 -0
  364. {evalview-0.8.0 → evalview-0.9.0}/tests/test_run_cmd_tags.py +0 -0
  365. {evalview-0.8.0 → evalview-0.9.0}/tests/test_run_guidance.py +0 -0
  366. {evalview-0.8.0 → evalview-0.9.0}/tests/test_runner.py +0 -0
  367. {evalview-0.8.0 → evalview-0.9.0}/tests/test_security.py +0 -0
  368. {evalview-0.8.0 → evalview-0.9.0}/tests/test_semantic_diff.py +0 -0
  369. {evalview-0.8.0 → evalview-0.9.0}/tests/test_simulation.py +0 -0
  370. {evalview-0.8.0 → evalview-0.9.0}/tests/test_snapshot_generated_workflow.py +0 -0
  371. {evalview-0.8.0 → evalview-0.9.0}/tests/test_snapshot_json_output.py +0 -0
  372. {evalview-0.8.0 → evalview-0.9.0}/tests/test_statistical_evaluator.py +0 -0
  373. {evalview-0.8.0 → evalview-0.9.0}/tests/test_telemetry_client.py +0 -0
  374. {evalview-0.8.0 → evalview-0.9.0}/tests/test_turn_coherence.py +0 -0
  375. {evalview-0.8.0 → evalview-0.9.0}/tests/test_types.py +0 -0
  376. {evalview-0.8.0 → evalview-0.9.0}/tests/test_validate_cmd.py +0 -0
  377. {evalview-0.8.0 → evalview-0.9.0}/tests/test_verdict.py +0 -0
  378. {evalview-0.8.0 → evalview-0.9.0}/tests/test_version_check.py +0 -0
  379. {evalview-0.8.0 → evalview-0.9.0}/tests/test_visualization_generators.py +0 -0
  380. {evalview-0.8.0 → evalview-0.9.0}/tests/test_week1_integration.py +0 -0
  381. {evalview-0.8.0 → evalview-0.9.0}/tests/test_week2_quarantine.py +0 -0
  382. {evalview-0.8.0 → evalview-0.9.0}/tests/test_week2_review_fixes.py +0 -0
  383. {evalview-0.8.0 → evalview-0.9.0}/tests/test_week2_since_and_comment.py +0 -0
@@ -0,0 +1,249 @@
1
+ Metadata-Version: 2.4
2
+ Name: evalview
3
+ Version: 0.9.0
4
+ Summary: Open-source testing and regression detection framework for AI agents. Golden baseline diffing, CI/CD integration, works with LangGraph, CrewAI, OpenAI, Anthropic Claude, HuggingFace, Ollama, and MCP.
5
+ Author-email: EvalView Team <hidai@evalview.com>
6
+ License-Expression: Apache-2.0
7
+ Project-URL: Homepage, https://evalview.com
8
+ Project-URL: Documentation, https://github.com/hidai25/eval-view#readme
9
+ Project-URL: Repository, https://github.com/hidai25/eval-view.git
10
+ Project-URL: Issue Tracker, https://github.com/hidai25/eval-view/issues
11
+ Project-URL: Changelog, https://github.com/hidai25/eval-view/blob/main/CHANGELOG.md
12
+ Keywords: ai,agents,testing,evaluation,llm,langchain,langgraph,crewai,openai,anthropic,claude,huggingface,ollama,mcp,multi-agent,pytest-ai,ai-agent-testing,llm-testing,agent-evaluation,regression-testing,golden-baseline,ci-cd-testing,yaml-testing,tool-calling,agent-regression,llm-regression,skill-testing,mcp-contract-testing,pass-at-k,non-deterministic-testing,ai-ci-cd
13
+ Classifier: Development Status :: 4 - Beta
14
+ Classifier: Intended Audience :: Developers
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.9
17
+ Classifier: Programming Language :: Python :: 3.10
18
+ Classifier: Programming Language :: Python :: 3.11
19
+ Classifier: Programming Language :: Python :: 3.12
20
+ Classifier: Topic :: Software Development :: Testing
21
+ Classifier: Topic :: Software Development :: Quality Assurance
22
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
23
+ Requires-Python: >=3.9
24
+ Description-Content-Type: text/markdown
25
+ License-File: LICENSE
26
+ License-File: NOTICE
27
+ Requires-Dist: click>=8.1.0
28
+ Requires-Dist: pydantic>=2.5.0
29
+ Requires-Dist: pyyaml>=6.0
30
+ Requires-Dist: openai>=1.101.0
31
+ Requires-Dist: anthropic>=0.39.0
32
+ Requires-Dist: rich>=13.7.0
33
+ Requires-Dist: prompt_toolkit>=3.0.0
34
+ Requires-Dist: httpx>=0.26.0
35
+ Requires-Dist: python-dateutil>=2.8.2
36
+ Requires-Dist: python-dotenv>=1.0.0
37
+ Requires-Dist: jinja2>=3.0
38
+ Requires-Dist: jsonschema>=4.0.0
39
+ Requires-Dist: llm-prices>=1.4.0
40
+ Requires-Dist: tomli>=2.0; python_version < "3.11"
41
+ Provides-Extra: reports
42
+ Requires-Dist: plotly>=5.0; extra == "reports"
43
+ Provides-Extra: watch
44
+ Requires-Dist: watchdog>=3.0; extra == "watch"
45
+ Provides-Extra: schedule
46
+ Requires-Dist: croniter>=2.0; extra == "schedule"
47
+ Provides-Extra: telemetry
48
+ Requires-Dist: posthog>=3.0.0; extra == "telemetry"
49
+ Provides-Extra: cohere
50
+ Requires-Dist: cohere>=5.0.0; extra == "cohere"
51
+ Provides-Extra: mistral
52
+ Requires-Dist: mistralai>=1.0.0; extra == "mistral"
53
+ Provides-Extra: pydantic-ai
54
+ Requires-Dist: pydantic-ai-slim>=0.0.30; python_version >= "3.10" and extra == "pydantic-ai"
55
+ Provides-Extra: all
56
+ Requires-Dist: plotly>=5.0; extra == "all"
57
+ Requires-Dist: watchdog>=3.0; extra == "all"
58
+ Requires-Dist: croniter>=2.0; extra == "all"
59
+ Requires-Dist: posthog>=3.0.0; extra == "all"
60
+ Requires-Dist: cohere>=5.0.0; extra == "all"
61
+ Requires-Dist: mistralai>=1.0.0; extra == "all"
62
+ Requires-Dist: pydantic-ai-slim>=0.0.30; python_version >= "3.10" and extra == "all"
63
+ Provides-Extra: dev
64
+ Requires-Dist: pytest>=7.4.0; extra == "dev"
65
+ Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
66
+ Requires-Dist: pytest-cov>=4.1.0; extra == "dev"
67
+ Requires-Dist: black==24.10.0; extra == "dev"
68
+ Requires-Dist: mypy>=1.7.0; extra == "dev"
69
+ Requires-Dist: types-PyYAML>=6.0; extra == "dev"
70
+ Requires-Dist: ruff>=0.1.0; extra == "dev"
71
+ Requires-Dist: jinja2>=3.0; extra == "dev"
72
+ Requires-Dist: plotly>=5.0; extra == "dev"
73
+ Requires-Dist: watchdog>=3.0; extra == "dev"
74
+ Requires-Dist: anthropic>=0.39.0; extra == "dev"
75
+ Requires-Dist: fastapi>=0.109.0; extra == "dev"
76
+ Requires-Dist: uvicorn>=0.27.0; extra == "dev"
77
+ Requires-Dist: pydantic-ai-slim>=0.0.30; python_version >= "3.10" and extra == "dev"
78
+ Dynamic: license-file
79
+
80
+ <!-- mcp-name: io.github.hidai25/evalview-mcp -->
81
+ <!-- keywords: AI agent testing, regression detection, golden baselines -->
82
+
83
+ <p align="center">
84
+ <img src="assets/logo.png" alt="EvalView" width="350">
85
+ <br>
86
+ <strong>Snapshot testing for AI agents.</strong><br>
87
+ Record what your agent does today. Get told when it silently changes.
88
+ </p>
89
+
90
+ <p align="center">
91
+ <a href="https://pypi.org/project/evalview/"><img src="https://img.shields.io/pypi/v/evalview.svg?label=release" alt="PyPI version"></a>
92
+ <a href="https://pypi.org/project/evalview/"><img src="https://img.shields.io/pypi/dm/evalview.svg?label=downloads" alt="PyPI downloads"></a>
93
+ <a href="https://github.com/hidai25/eval-view/actions/workflows/ci.yml"><img src="https://github.com/hidai25/eval-view/actions/workflows/ci.yml/badge.svg?branch=main" alt="Package CI"></a>
94
+ <a href="https://github.com/hidai25/eval-view/actions/workflows/dogfood.yml"><img src="https://github.com/hidai25/eval-view/actions/workflows/dogfood.yml/badge.svg?branch=main" alt="Core Dogfood"></a>
95
+ <a href="https://github.com/hidai25/eval-view/actions/workflows/dogfood-live.yml"><img src="https://github.com/hidai25/eval-view/actions/workflows/dogfood-live.yml/badge.svg?branch=main&amp;event=workflow_dispatch" alt="Live Provider Checks (manual)"></a>
96
+ <a href="https://github.com/hidai25/eval-view/stargazers"><img src="https://img.shields.io/github/stars/hidai25/eval-view?style=social" alt="GitHub stars"></a>
97
+ <a href="https://opensource.org/licenses/Apache-2.0"><img src="https://img.shields.io/badge/License-Apache_2.0-blue.svg" alt="License"></a>
98
+ </p>
99
+
100
+ ---
101
+
102
+ Your agent returns `200` and looks fine. But a model update, a provider change, or a one-line prompt edit just made it skip a clarification, call the wrong tool, or quietly drop output quality. Your tests still pass. Your users notice before you do.
103
+
104
+ **EvalView snapshots your agent's behavior — the tools it calls, in what order, with what output — and tells you the moment that behavior changes.** Like Jest snapshots, but for tool-calling, multi-turn agents.
105
+
106
+ [![demo.gif](assets/demo.gif)](https://github.com/user-attachments/assets/96d8b5f7-3561-44a1-86a4-270fb0d1d8a6)
107
+
108
+ <sub>↑ 30-second live demo — no API key needed</sub>
109
+
110
+ ## Quick Start
111
+
112
+ > **OpenAI adapter migration:** OpenAI shut down the Assistants API on **August 26, 2026**.
113
+ > EvalView 0.9.0 uses the Responses API. If you use `openai-assistants`, migrate your
114
+ > model, instructions and tools or configure a dashboard Prompt before running checks.
115
+ > [OpenAI migration guide](docs/OPENAI_MIGRATION.md)
116
+
117
+ ```bash
118
+ pip install evalview
119
+ ```
120
+
121
+ ```bash
122
+ evalview snapshot # Record your agent's current behavior as the baseline
123
+ evalview check # After any change, diff against the baseline
124
+ ```
125
+
126
+ That's the whole loop. `check` returns one of:
127
+
128
+ ```
129
+ ✓ login-flow PASSED behavior matches baseline
130
+ ⚠ refund-request TOOLS_CHANGED called a different tool, or in a different order
131
+ ✗ billing-dispute REGRESSION score dropped — output quality fell
132
+ ```
133
+
134
+ It diffs the **whole trajectory** — tool names, parameters, and order — not just the final string. The deterministic tool + sequence diff runs offline, with no API key. Add an LLM judge only when you want output-quality scoring.
135
+
136
+ Executing your agent can still incur backend API charges: `--no-judge` skips the
137
+ judge, not those calls. Embedding-based semantic comparison is opt-in.
138
+
139
+ No agent yet? See it work in 30 seconds:
140
+
141
+ ```bash
142
+ evalview demo
143
+ ```
144
+
145
+ ## Why snapshot testing (and not assertions)?
146
+
147
+ Most eval tools ask you to *write down what "good" looks like* — assertions, metrics, rubrics. That's a lot of upfront work, and you can only catch the failures you thought to assert.
148
+
149
+ EvalView inverts it: **it records what your agent actually does now, and flags any drift from that.** You catch regressions you never anticipated, with zero assertions written. When the new behavior is correct, `evalview snapshot` accepts it as the new baseline — same as updating a snapshot in Jest.
150
+
151
+ | | EvalView | Assertion-based eval tools |
152
+ |---|---|---|
153
+ | Setup | Record current behavior | Write assertions/metrics first |
154
+ | Catches | Any drift from baseline | Only what you asserted |
155
+ | Non-determinism | Multi-variant baselines (up to 5 valid paths) | You handle it |
156
+ | Unit of comparison | Full tool-call trajectory | Usually final output |
157
+
158
+ This makes EvalView a **merge-time regression gate**, which is a different job from observability (Langfuse, LangSmith) or metric scoring (promptfoo, DeepEval, Braintrust). Many teams run one of those for visibility **and** EvalView as the gate. [Honest comparisons →](docs/COMPARISONS.md)
159
+
160
+ ## EvalView tests itself in public, every day
161
+
162
+ Every day at 09:00 UTC, on pull requests, and on pushes to main,
163
+ [Core Dogfood](.github/workflows/dogfood.yml) exercises the non-live test suite,
164
+ type checks, local mock-agent `snapshot` / `check`, `evalview demo`, end-to-end
165
+ flows, and an `evalview monitor` smoke test. It uses no paid API credentials and
166
+ makes no paid inference calls. GitHub runner usage is separate.
167
+
168
+ [Live Provider Checks](.github/workflows/dogfood-live.yml) test the real evaluator and
169
+ chat assistant only when a maintainer explicitly opts into paid API use on main.
170
+ They have no automatic schedule. Their badge records the last manual run; a green
171
+ core badge does not establish live-provider health or rule out provider drift.
172
+
173
+ Package CI, core dogfood, and live checks have separate badges. Failed or incomplete
174
+ checks remain visible within their scope, with logs and reports preserved as
175
+ artifacts. Rolling issues use separate `dogfood-core` and `dogfood-live` labels.
176
+ A provider outage, exhausted quota, or missing credential means live health is
177
+ unavailable; it does not prove an agent regression.
178
+
179
+ The historical [incident #264](https://github.com/hidai25/eval-view/issues/264)
180
+ remains available for maintainer review of fresh evidence from both scopes. Neither
181
+ workflow automatically closes it. Trust warnings are evidence to investigate,
182
+ not proof of gaming or of a particular root cause.
183
+
184
+ [Core runs →](https://github.com/hidai25/eval-view/actions/workflows/dogfood.yml) · [Manual live runs →](https://github.com/hidai25/eval-view/actions/workflows/dogfood-live.yml) · [Run and triage guide →](docs/INTERNAL_DOGFOODING.md#daily-failure-triage)
185
+
186
+ ## CI: block regressions in every PR
187
+
188
+ ```yaml
189
+ # .github/workflows/evalview.yml
190
+ name: EvalView
191
+ on: [pull_request]
192
+ jobs:
193
+ agent-check:
194
+ runs-on: ubuntu-latest
195
+ permissions: { pull-requests: write }
196
+ steps:
197
+ - uses: actions/checkout@v4
198
+ - uses: hidai25/eval-view@v0.9.0
199
+ with:
200
+ openai-api-key: ${{ secrets.OPENAI_API_KEY }}
201
+ ```
202
+
203
+ You get a PR comment with the diff, cost/latency deltas, and a pass/fail gate. [CI/CD guide →](docs/CI_CD.md)
204
+
205
+ ## Works with your stack
206
+
207
+ LangGraph · CrewAI · OpenAI · Claude · Mistral · Ollama · MCP · **any HTTP API**.
208
+
209
+ ```bash
210
+ evalview check --agent http://localhost:8000/invoke
211
+ ```
212
+
213
+ [Framework details →](docs/FRAMEWORK_SUPPORT.md)
214
+
215
+ ## Use it as a library
216
+
217
+ ```python
218
+ from evalview import gate
219
+
220
+ result = gate(test_dir="tests/")
221
+ result.passed # bool
222
+ result.diffs # per-test scores and tool diffs
223
+ ```
224
+
225
+ [Python API →](docs/CLI_REFERENCE.md#python-api)
226
+
227
+ ## More
228
+
229
+ EvalView also does multi-turn testing, statistical/pass@k runs, record/replay cassettes, model-drift canaries, production monitoring with Slack alerts, and auto-generated regression tests from incidents. These are power-user features — start with `snapshot` and `check`, reach for the rest when you need them.
230
+
231
+ → [Full feature reference](docs/CLI_REFERENCE.md) · [Getting Started](docs/GETTING_STARTED.md) · [FAQ](docs/FAQ.md)
232
+
233
+ → [Documentation index](docs/README.md) · [OpenAI migration](docs/OPENAI_MIGRATION.md) · [Release process](docs/RELEASING.md)
234
+
235
+ ### Why I built EvalView
236
+
237
+ An agent that looked successful kept pulling entire documents into its context and made one question cost $42.93. That experience led me to build EvalView. I wrote about it in [“I Was Running an AI Casino. Then I Started Writing Tests for My Agents”](https://medium.com/@hidaibarmor/i-was-running-an-ai-casino-then-i-started-writing-tests-for-my-agents-93cb3468ce1e). The December 2025 post is the origin story; use the current docs for setup and commands.
238
+
239
+ ## Contributing
240
+
241
+ This is a young project built mostly by one developer. Issues, PRs, and "I tried it and X was confusing" feedback are all genuinely valuable.
242
+
243
+ - [Open an issue](https://github.com/hidai25/eval-view/issues) · [Discussions](https://github.com/hidai25/eval-view/discussions) · [CONTRIBUTING.md](CONTRIBUTING.md)
244
+
245
+ **License:** Apache 2.0
246
+
247
+ ---
248
+
249
+ [![Star History Chart](https://api.star-history.com/svg?repos=hidai25/eval-view&type=Date)](https://star-history.com/#hidai25/eval-view&Date)
@@ -0,0 +1,170 @@
1
+ <!-- mcp-name: io.github.hidai25/evalview-mcp -->
2
+ <!-- keywords: AI agent testing, regression detection, golden baselines -->
3
+
4
+ <p align="center">
5
+ <img src="assets/logo.png" alt="EvalView" width="350">
6
+ <br>
7
+ <strong>Snapshot testing for AI agents.</strong><br>
8
+ Record what your agent does today. Get told when it silently changes.
9
+ </p>
10
+
11
+ <p align="center">
12
+ <a href="https://pypi.org/project/evalview/"><img src="https://img.shields.io/pypi/v/evalview.svg?label=release" alt="PyPI version"></a>
13
+ <a href="https://pypi.org/project/evalview/"><img src="https://img.shields.io/pypi/dm/evalview.svg?label=downloads" alt="PyPI downloads"></a>
14
+ <a href="https://github.com/hidai25/eval-view/actions/workflows/ci.yml"><img src="https://github.com/hidai25/eval-view/actions/workflows/ci.yml/badge.svg?branch=main" alt="Package CI"></a>
15
+ <a href="https://github.com/hidai25/eval-view/actions/workflows/dogfood.yml"><img src="https://github.com/hidai25/eval-view/actions/workflows/dogfood.yml/badge.svg?branch=main" alt="Core Dogfood"></a>
16
+ <a href="https://github.com/hidai25/eval-view/actions/workflows/dogfood-live.yml"><img src="https://github.com/hidai25/eval-view/actions/workflows/dogfood-live.yml/badge.svg?branch=main&amp;event=workflow_dispatch" alt="Live Provider Checks (manual)"></a>
17
+ <a href="https://github.com/hidai25/eval-view/stargazers"><img src="https://img.shields.io/github/stars/hidai25/eval-view?style=social" alt="GitHub stars"></a>
18
+ <a href="https://opensource.org/licenses/Apache-2.0"><img src="https://img.shields.io/badge/License-Apache_2.0-blue.svg" alt="License"></a>
19
+ </p>
20
+
21
+ ---
22
+
23
+ Your agent returns `200` and looks fine. But a model update, a provider change, or a one-line prompt edit just made it skip a clarification, call the wrong tool, or quietly drop output quality. Your tests still pass. Your users notice before you do.
24
+
25
+ **EvalView snapshots your agent's behavior — the tools it calls, in what order, with what output — and tells you the moment that behavior changes.** Like Jest snapshots, but for tool-calling, multi-turn agents.
26
+
27
+ [![demo.gif](assets/demo.gif)](https://github.com/user-attachments/assets/96d8b5f7-3561-44a1-86a4-270fb0d1d8a6)
28
+
29
+ <sub>↑ 30-second live demo — no API key needed</sub>
30
+
31
+ ## Quick Start
32
+
33
+ > **OpenAI adapter migration:** OpenAI shut down the Assistants API on **August 26, 2026**.
34
+ > EvalView 0.9.0 uses the Responses API. If you use `openai-assistants`, migrate your
35
+ > model, instructions and tools or configure a dashboard Prompt before running checks.
36
+ > [OpenAI migration guide](docs/OPENAI_MIGRATION.md)
37
+
38
+ ```bash
39
+ pip install evalview
40
+ ```
41
+
42
+ ```bash
43
+ evalview snapshot # Record your agent's current behavior as the baseline
44
+ evalview check # After any change, diff against the baseline
45
+ ```
46
+
47
+ That's the whole loop. `check` returns one of:
48
+
49
+ ```
50
+ ✓ login-flow PASSED behavior matches baseline
51
+ ⚠ refund-request TOOLS_CHANGED called a different tool, or in a different order
52
+ ✗ billing-dispute REGRESSION score dropped — output quality fell
53
+ ```
54
+
55
+ It diffs the **whole trajectory** — tool names, parameters, and order — not just the final string. The deterministic tool + sequence diff runs offline, with no API key. Add an LLM judge only when you want output-quality scoring.
56
+
57
+ Executing your agent can still incur backend API charges: `--no-judge` skips the
58
+ judge, not those calls. Embedding-based semantic comparison is opt-in.
59
+
60
+ No agent yet? See it work in 30 seconds:
61
+
62
+ ```bash
63
+ evalview demo
64
+ ```
65
+
66
+ ## Why snapshot testing (and not assertions)?
67
+
68
+ Most eval tools ask you to *write down what "good" looks like* — assertions, metrics, rubrics. That's a lot of upfront work, and you can only catch the failures you thought to assert.
69
+
70
+ EvalView inverts it: **it records what your agent actually does now, and flags any drift from that.** You catch regressions you never anticipated, with zero assertions written. When the new behavior is correct, `evalview snapshot` accepts it as the new baseline — same as updating a snapshot in Jest.
71
+
72
+ | | EvalView | Assertion-based eval tools |
73
+ |---|---|---|
74
+ | Setup | Record current behavior | Write assertions/metrics first |
75
+ | Catches | Any drift from baseline | Only what you asserted |
76
+ | Non-determinism | Multi-variant baselines (up to 5 valid paths) | You handle it |
77
+ | Unit of comparison | Full tool-call trajectory | Usually final output |
78
+
79
+ This makes EvalView a **merge-time regression gate**, which is a different job from observability (Langfuse, LangSmith) or metric scoring (promptfoo, DeepEval, Braintrust). Many teams run one of those for visibility **and** EvalView as the gate. [Honest comparisons →](docs/COMPARISONS.md)
80
+
81
+ ## EvalView tests itself in public, every day
82
+
83
+ Every day at 09:00 UTC, on pull requests, and on pushes to main,
84
+ [Core Dogfood](.github/workflows/dogfood.yml) exercises the non-live test suite,
85
+ type checks, local mock-agent `snapshot` / `check`, `evalview demo`, end-to-end
86
+ flows, and an `evalview monitor` smoke test. It uses no paid API credentials and
87
+ makes no paid inference calls. GitHub runner usage is separate.
88
+
89
+ [Live Provider Checks](.github/workflows/dogfood-live.yml) test the real evaluator and
90
+ chat assistant only when a maintainer explicitly opts into paid API use on main.
91
+ They have no automatic schedule. Their badge records the last manual run; a green
92
+ core badge does not establish live-provider health or rule out provider drift.
93
+
94
+ Package CI, core dogfood, and live checks have separate badges. Failed or incomplete
95
+ checks remain visible within their scope, with logs and reports preserved as
96
+ artifacts. Rolling issues use separate `dogfood-core` and `dogfood-live` labels.
97
+ A provider outage, exhausted quota, or missing credential means live health is
98
+ unavailable; it does not prove an agent regression.
99
+
100
+ The historical [incident #264](https://github.com/hidai25/eval-view/issues/264)
101
+ remains available for maintainer review of fresh evidence from both scopes. Neither
102
+ workflow automatically closes it. Trust warnings are evidence to investigate,
103
+ not proof of gaming or of a particular root cause.
104
+
105
+ [Core runs →](https://github.com/hidai25/eval-view/actions/workflows/dogfood.yml) · [Manual live runs →](https://github.com/hidai25/eval-view/actions/workflows/dogfood-live.yml) · [Run and triage guide →](docs/INTERNAL_DOGFOODING.md#daily-failure-triage)
106
+
107
+ ## CI: block regressions in every PR
108
+
109
+ ```yaml
110
+ # .github/workflows/evalview.yml
111
+ name: EvalView
112
+ on: [pull_request]
113
+ jobs:
114
+ agent-check:
115
+ runs-on: ubuntu-latest
116
+ permissions: { pull-requests: write }
117
+ steps:
118
+ - uses: actions/checkout@v4
119
+ - uses: hidai25/eval-view@v0.9.0
120
+ with:
121
+ openai-api-key: ${{ secrets.OPENAI_API_KEY }}
122
+ ```
123
+
124
+ You get a PR comment with the diff, cost/latency deltas, and a pass/fail gate. [CI/CD guide →](docs/CI_CD.md)
125
+
126
+ ## Works with your stack
127
+
128
+ LangGraph · CrewAI · OpenAI · Claude · Mistral · Ollama · MCP · **any HTTP API**.
129
+
130
+ ```bash
131
+ evalview check --agent http://localhost:8000/invoke
132
+ ```
133
+
134
+ [Framework details →](docs/FRAMEWORK_SUPPORT.md)
135
+
136
+ ## Use it as a library
137
+
138
+ ```python
139
+ from evalview import gate
140
+
141
+ result = gate(test_dir="tests/")
142
+ result.passed # bool
143
+ result.diffs # per-test scores and tool diffs
144
+ ```
145
+
146
+ [Python API →](docs/CLI_REFERENCE.md#python-api)
147
+
148
+ ## More
149
+
150
+ EvalView also does multi-turn testing, statistical/pass@k runs, record/replay cassettes, model-drift canaries, production monitoring with Slack alerts, and auto-generated regression tests from incidents. These are power-user features — start with `snapshot` and `check`, reach for the rest when you need them.
151
+
152
+ → [Full feature reference](docs/CLI_REFERENCE.md) · [Getting Started](docs/GETTING_STARTED.md) · [FAQ](docs/FAQ.md)
153
+
154
+ → [Documentation index](docs/README.md) · [OpenAI migration](docs/OPENAI_MIGRATION.md) · [Release process](docs/RELEASING.md)
155
+
156
+ ### Why I built EvalView
157
+
158
+ An agent that looked successful kept pulling entire documents into its context and made one question cost $42.93. That experience led me to build EvalView. I wrote about it in [“I Was Running an AI Casino. Then I Started Writing Tests for My Agents”](https://medium.com/@hidaibarmor/i-was-running-an-ai-casino-then-i-started-writing-tests-for-my-agents-93cb3468ce1e). The December 2025 post is the origin story; use the current docs for setup and commands.
159
+
160
+ ## Contributing
161
+
162
+ This is a young project built mostly by one developer. Issues, PRs, and "I tried it and X was confusing" feedback are all genuinely valuable.
163
+
164
+ - [Open an issue](https://github.com/hidai25/eval-view/issues) · [Discussions](https://github.com/hidai25/eval-view/discussions) · [CONTRIBUTING.md](CONTRIBUTING.md)
165
+
166
+ **License:** Apache 2.0
167
+
168
+ ---
169
+
170
+ [![Star History Chart](https://api.star-history.com/svg?repos=hidai25/eval-view&type=Date)](https://star-history.com/#hidai25/eval-view&Date)
@@ -1,6 +1,6 @@
1
1
  """EvalView - Testing framework for multi-step AI agents."""
2
2
 
3
- __version__ = "0.7.0"
3
+ __version__ = "0.9.0"
4
4
 
5
5
  # Public API — importable as ``from evalview import gate``
6
6
  from evalview.api import gate, gate_async, GateResult, DiffStatus # noqa: F401
@@ -49,17 +49,26 @@ class CohereAdapter(AgentAdapter):
49
49
  async with tracer.start_span_async("Cohere Agent", SpanKind.AGENT):
50
50
  api_start = datetime.now()
51
51
 
52
- # Send request to Cohere
52
+ # Send request to Cohere. The import is safe here: __init__ already
53
+ # verified the optional cohere package is installed.
54
+ from cohere import UserChatMessageV2
55
+
53
56
  response = await self.client.chat(
54
57
  model=self.model,
55
- messages=[{"role": "user", "content": query}]
58
+ messages=[UserChatMessageV2(content=query)]
56
59
  )
57
-
60
+
58
61
  api_end = datetime.now()
59
62
  api_latency = (api_end - api_start).total_seconds() * 1000
60
63
 
61
- # Extract AI's response text
62
- final_answer = response.message.content[0].text
64
+ # Extract the AI's response text. content is Optional, and items may
65
+ # be thinking blocks that carry no text payload.
66
+ final_answer = ""
67
+ for item in response.message.content or []:
68
+ text = getattr(item, "text", None)
69
+ if text:
70
+ final_answer = text
71
+ break
63
72
 
64
73
  # 2. Extract Token Usage (Changes for compatibility with V1 and V2 SDKs)
65
74
  input_tokens = 0
@@ -262,8 +262,11 @@ class CrewAINativeAdapter(AgentAdapter):
262
262
 
263
263
  rationale = RationaleCollector()
264
264
  last_agent: Optional[str] = None
265
- for i, (st, tc) in enumerate(zip(steps, tool_calls or [None] * len(steps))):
266
- agent_name = (tc or {}).get("agent") if tc else None
265
+ calls_by_step: List[Optional[Dict[str, Any]]] = (
266
+ list(tool_calls) if tool_calls else [None] * len(steps)
267
+ )
268
+ for i, (st, call) in enumerate(zip(steps, calls_by_step)):
269
+ agent_name = call.get("agent") if call else None
267
270
  if agent_name and agent_name != last_agent:
268
271
  rationale.capture_branch(
269
272
  step_id=st.step_id,
@@ -317,9 +317,13 @@ class GooseAdapter(AgentAdapter):
317
317
  if tokens_data:
318
318
  if isinstance(tokens_data, dict):
319
319
  total_tokens = TokenUsage(
320
- input_tokens=tokens_data.get("input", tokens_data.get("input_tokens", 0)),
321
- output_tokens=tokens_data.get("output", tokens_data.get("output_tokens", 0)),
322
- cached_tokens=tokens_data.get("cached", 0),
320
+ input_tokens=int(
321
+ tokens_data.get("input", tokens_data.get("input_tokens", 0)) or 0
322
+ ),
323
+ output_tokens=int(
324
+ tokens_data.get("output", tokens_data.get("output_tokens", 0)) or 0
325
+ ),
326
+ cached_tokens=int(tokens_data.get("cached", 0) or 0),
323
327
  )
324
328
  elif isinstance(tokens_data, int):
325
329
  total_tokens = TokenUsage(output_tokens=tokens_data)
@@ -153,8 +153,12 @@ class HTTPAdapter(AgentAdapter):
153
153
  input_tokens = 0
154
154
  output_tokens = 0
155
155
  if isinstance(tokens_data, dict):
156
- input_tokens = tokens_data.get("input", tokens_data.get("input_tokens", 0))
157
- output_tokens = tokens_data.get("output", tokens_data.get("output_tokens", 0))
156
+ input_tokens = int(
157
+ tokens_data.get("input", tokens_data.get("input_tokens", 0)) or 0
158
+ )
159
+ output_tokens = int(
160
+ tokens_data.get("output", tokens_data.get("output_tokens", 0)) or 0
161
+ )
158
162
 
159
163
  tracer.record_llm_call(
160
164
  model=model_name,
@@ -250,9 +254,15 @@ class HTTPAdapter(AgentAdapter):
250
254
  if isinstance(tokens_data, dict):
251
255
  # Nested format: {"input": 100, "output": 500, "cached": 50}
252
256
  total_tokens = TokenUsage(
253
- input_tokens=tokens_data.get("input", tokens_data.get("input_tokens", 0)),
254
- output_tokens=tokens_data.get("output", tokens_data.get("output_tokens", 0)),
255
- cached_tokens=tokens_data.get("cached", tokens_data.get("cached_tokens", 0)),
257
+ input_tokens=int(
258
+ tokens_data.get("input", tokens_data.get("input_tokens", 0)) or 0
259
+ ),
260
+ output_tokens=int(
261
+ tokens_data.get("output", tokens_data.get("output_tokens", 0)) or 0
262
+ ),
263
+ cached_tokens=int(
264
+ tokens_data.get("cached", tokens_data.get("cached_tokens", 0)) or 0
265
+ ),
256
266
  )
257
267
  elif isinstance(tokens_data, int):
258
268
  # Simple total: {"tokens": 1500}
@@ -128,6 +128,7 @@ class LangGraphAdapter(AgentAdapter):
128
128
 
129
129
  # Step 2: Create run with streaming
130
130
  input_key = context.get("input_key", self.input_key)
131
+ run_input: Dict[str, Any]
131
132
  if input_key:
132
133
  run_input = {input_key: query}
133
134
  else:
@@ -559,7 +560,7 @@ class LangGraphAdapter(AgentAdapter):
559
560
  return StepTrace(
560
561
  step_id=step_data.get("id", f"step-{index}"),
561
562
  step_name=step_data.get("name", f"Step {index + 1}"),
562
- tool_name=step_data.get("tool"),
563
+ tool_name=step_data.get("tool") or "unknown",
563
564
  parameters=step_data.get("parameters", {}),
564
565
  output=step_data.get("output"),
565
566
  success=step_data.get("success", True),
@@ -579,7 +580,7 @@ class LangGraphAdapter(AgentAdapter):
579
580
  return StepTrace(
580
581
  step_id=event.get("id", f"step-{datetime.now().timestamp()}"),
581
582
  step_name=content[:50] if content else "Step",
582
- tool_name=event.get("tool"),
583
+ tool_name=event.get("tool") or "unknown",
583
584
  parameters=event.get("parameters", {}),
584
585
  output=content,
585
586
  success=True,
@@ -344,6 +344,8 @@ class MCPAdapter(AgentAdapter):
344
344
  self, process: asyncio.subprocess.Process, method: str, params: Dict
345
345
  ) -> Dict:
346
346
  """Send JSON-RPC request via stdio."""
347
+ # Process is always spawned with stdin/stdout=PIPE
348
+ assert process.stdin is not None and process.stdout is not None
347
349
  self._request_id += 1
348
350
  request = {
349
351
  "jsonrpc": "2.0",
@@ -377,6 +379,8 @@ class MCPAdapter(AgentAdapter):
377
379
  self, process: asyncio.subprocess.Process, method: str, params: Dict
378
380
  ) -> None:
379
381
  """Send JSON-RPC notification (no response expected)."""
382
+ # Process is always spawned with stdin=PIPE
383
+ assert process.stdin is not None
380
384
  notification = {
381
385
  "jsonrpc": "2.0",
382
386
  "method": method,
@@ -1,10 +1,11 @@
1
1
  import os
2
2
  import logging
3
3
  from datetime import datetime
4
- from typing import Any, Optional, Dict
4
+ from typing import Any, List, Optional, Dict
5
5
 
6
6
  try:
7
- from mistralai import Mistral
7
+ from mistralai import Mistral, UserMessage
8
+ from mistralai.models import Messages
8
9
  except ImportError:
9
10
  raise ImportError(
10
11
  "The mistralai package is required for the Mistral adapter. "
@@ -51,18 +52,28 @@ class MistralAdapter(AgentAdapter):
51
52
  api_start = datetime.now()
52
53
 
53
54
  # Send request to Mistral
55
+ messages: List[Messages] = [UserMessage(content=query)]
54
56
  response = await self.client.chat.complete_async(
55
57
  model=self.model,
56
- messages=[{"role": "user", "content": query}]
58
+ messages=messages
57
59
  )
58
-
60
+
59
61
  api_end = datetime.now()
60
62
  api_latency = (api_end - api_start).total_seconds() * 1000
61
63
 
62
- # Extract AI's response text and token usage
63
- final_answer = response.choices[0].message.content
64
- input_tokens = response.usage.prompt_tokens
65
- output_tokens = response.usage.completion_tokens
64
+ # Extract the AI's response text. content may be a plain string or a
65
+ # list of chunks (text, think, image, ...); join the text-bearing ones.
66
+ content = response.choices[0].message.content
67
+ if isinstance(content, str):
68
+ final_answer = content
69
+ elif isinstance(content, list):
70
+ final_answer = "".join(getattr(chunk, "text", "") or "" for chunk in content)
71
+ else:
72
+ final_answer = ""
73
+
74
+ # Token counts are Optional in the SDK response
75
+ input_tokens = response.usage.prompt_tokens or 0
76
+ output_tokens = response.usage.completion_tokens or 0
66
77
 
67
78
  # Calculate Cost
68
79
  total_cost = calculate_cost(
@@ -101,10 +101,10 @@ class OllamaAdapter(AgentAdapter):
101
101
  finish_reason = data.get("choices", [{}])[0].get("finish_reason", "stop")
102
102
 
103
103
  # Build token usage
104
+ # total_tokens is a computed property on TokenUsage, not a field
104
105
  token_usage = TokenUsage(
105
106
  input_tokens=usage.get("prompt_tokens", 0),
106
107
  output_tokens=usage.get("completion_tokens", 0),
107
- total_tokens=usage.get("total_tokens", 0),
108
108
  )
109
109
 
110
110
  # Record LLM call span