evalview 0.8.0__tar.gz → 0.9.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- evalview-0.9.0/PKG-INFO +249 -0
- evalview-0.9.0/README.md +170 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/__init__.py +1 -1
- {evalview-0.8.0 → evalview-0.9.0}/evalview/adapters/cohere_adapter.py +14 -5
- {evalview-0.8.0 → evalview-0.9.0}/evalview/adapters/crewai_native_adapter.py +5 -2
- {evalview-0.8.0 → evalview-0.9.0}/evalview/adapters/goose_adapter.py +7 -3
- {evalview-0.8.0 → evalview-0.9.0}/evalview/adapters/http_adapter.py +15 -5
- {evalview-0.8.0 → evalview-0.9.0}/evalview/adapters/langgraph_adapter.py +3 -2
- {evalview-0.8.0 → evalview-0.9.0}/evalview/adapters/mcp_adapter.py +4 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/adapters/mistral_adapter.py +19 -8
- {evalview-0.8.0 → evalview-0.9.0}/evalview/adapters/ollama_adapter.py +1 -1
- evalview-0.9.0/evalview/adapters/openai_assistants_adapter.py +473 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/adapters/openclaw_adapter.py +7 -3
- {evalview-0.8.0 → evalview-0.9.0}/evalview/adapters/pydantic_ai_adapter.py +102 -22
- {evalview-0.8.0 → evalview-0.9.0}/evalview/adapters/registry.py +7 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/adapters/tapescope_adapter.py +2 -2
- evalview-0.9.0/evalview/adapters/vercel_ai_adapter.py +223 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/api.py +46 -37
- {evalview-0.8.0 → evalview-0.9.0}/evalview/chat.py +1 -1
- {evalview-0.8.0 → evalview-0.9.0}/evalview/chat_commands.py +1 -1
- {evalview-0.8.0 → evalview-0.9.0}/evalview/chat_prompt.py +37 -30
- {evalview-0.8.0 → evalview-0.9.0}/evalview/chat_session.py +43 -27
- {evalview-0.8.0 → evalview-0.9.0}/evalview/ci/comment.py +6 -3
- evalview-0.9.0/evalview/cloud/auth.py +125 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/cloud/push.py +66 -16
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/_check_verdict.py +9 -1
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/badge_cmd.py +37 -53
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/check_cmd.py +292 -95
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/check_display.py +69 -5
- evalview-0.9.0/evalview/commands/cloud_cmd.py +254 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/drift_cmd.py +2 -2
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/listing_cmd.py +1 -1
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/monitor_cmd.py +57 -26
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/run/_adapters.py +30 -7
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/run/_reporter.py +1 -1
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/shared.py +52 -64
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/snapshot_cmd.py +27 -20
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/trends_cmd.py +16 -1
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/watch_cmd.py +102 -16
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/adapter_factory.py +16 -1
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/chaos.py +3 -3
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/config.py +16 -12
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/diff.py +16 -2
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/drift_tracker.py +44 -37
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/fleet.py +3 -3
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/freshness.py +9 -23
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/goal_drift.py +5 -18
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/golden.py +106 -20
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/healing.py +32 -12
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/loader.py +29 -1
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/noise_tracker.py +7 -3
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/pricing.py +42 -15
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/recommendations.py +9 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/retrieval_lineage.py +5 -16
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/root_cause.py +10 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/runner.py +1 -1
- evalview-0.9.0/evalview/core/text.py +23 -0
- evalview-0.9.0/evalview/core/trends.py +91 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/types.py +43 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/verdict.py +10 -2
- {evalview-0.8.0 → evalview-0.9.0}/evalview/evaluators/evaluator.py +34 -18
- {evalview-0.8.0 → evalview-0.9.0}/evalview/evaluators/hallucination_evaluator.py +1 -1
- {evalview-0.8.0 → evalview-0.9.0}/evalview/evaluators/output_evaluator.py +15 -21
- {evalview-0.8.0 → evalview-0.9.0}/evalview/evaluators/sequence_evaluator.py +5 -5
- {evalview-0.8.0 → evalview-0.9.0}/evalview/evaluators/tool_call_evaluator.py +40 -4
- {evalview-0.8.0 → evalview-0.9.0}/evalview/mcp_server.py +226 -88
- {evalview-0.8.0 → evalview-0.9.0}/evalview/openclaw.py +12 -1
- evalview-0.9.0/evalview/py.typed +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/pytest_plugin.py +1 -1
- {evalview-0.8.0 → evalview-0.9.0}/evalview/reporters/_console_coverage.py +5 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/reporters/_console_statistical.py +4 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/reporters/console_reporter.py +3 -2
- {evalview-0.8.0 → evalview-0.9.0}/evalview/reporters/html_reporter.py +9 -8
- {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/adapters/__init__.py +1 -1
- evalview-0.9.0/evalview/skills/adapters/openai_assistants_adapter.py +531 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/adapters/registry.py +1 -1
- {evalview-0.8.0 → evalview-0.9.0}/evalview/tracking/__init__.py +2 -1
- {evalview-0.8.0 → evalview-0.9.0}/evalview/tracking/regression.py +61 -5
- {evalview-0.8.0 → evalview-0.9.0}/evalview/visualization/generators.py +10 -2
- evalview-0.9.0/evalview.egg-info/PKG-INFO +249 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview.egg-info/SOURCES.txt +23 -1
- {evalview-0.8.0 → evalview-0.9.0}/evalview.egg-info/requires.txt +13 -1
- {evalview-0.8.0 → evalview-0.9.0}/pyproject.toml +22 -14
- evalview-0.9.0/tests/test_action_check_status.py +61 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_api.py +7 -4
- evalview-0.9.0/tests/test_chat_session.py +84 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_check_cmd.py +36 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_check_pipeline.py +190 -41
- evalview-0.9.0/tests/test_ci_provider_isolation.py +126 -0
- evalview-0.9.0/tests/test_cloud_auth_and_push.py +203 -0
- evalview-0.9.0/tests/test_documentation_links.py +87 -0
- evalview-0.9.0/tests/test_dogfood_ci.py +204 -0
- evalview-0.9.0/tests/test_dogfood_harness.py +122 -0
- evalview-0.9.0/tests/test_dogfood_issue_routing.py +112 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_drift_tracker.py +63 -1
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_evaluators.py +163 -3
- evalview-0.9.0/tests/test_forbidden_tool_gate.py +140 -0
- evalview-0.9.0/tests/test_incomplete_gate.py +210 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_loader.py +8 -2
- evalview-0.9.0/tests/test_loader_skill_suites.py +111 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_mcp_server.py +339 -22
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_monitor.py +52 -0
- evalview-0.9.0/tests/test_openai_responses_adapter.py +555 -0
- evalview-0.9.0/tests/test_pricing.py +74 -0
- evalview-0.9.0/tests/test_pydantic_ai_adapter.py +126 -0
- evalview-0.9.0/tests/test_regression_tracker.py +279 -0
- evalview-0.9.0/tests/test_release_checks.py +167 -0
- evalview-0.9.0/tests/test_snapshot_reset.py +202 -0
- evalview-0.9.0/tests/test_vercel_ai_adapter.py +357 -0
- evalview-0.9.0/tests/test_watch_cmd.py +165 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_week3_progress_and_drift.py +2 -0
- evalview-0.8.0/PKG-INFO +0 -1063
- evalview-0.8.0/README.md +0 -989
- evalview-0.8.0/evalview/adapters/openai_assistants_adapter.py +0 -476
- evalview-0.8.0/evalview/cloud/auth.py +0 -80
- evalview-0.8.0/evalview/cloud/client.py +0 -175
- evalview-0.8.0/evalview/commands/cloud_cmd.py +0 -179
- evalview-0.8.0/evalview/skills/adapters/openai_assistants_adapter.py +0 -637
- evalview-0.8.0/evalview.egg-info/PKG-INFO +0 -1063
- {evalview-0.8.0 → evalview-0.9.0}/LICENSE +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/NOTICE +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/__main__.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/adapters/__init__.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/adapters/aider_adapter.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/adapters/anthropic_adapter.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/adapters/base.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/adapters/crewai_adapter.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/adapters/huggingface_adapter.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/adapters/opencode_adapter.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/benchmarks/__init__.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/benchmarks/canary/README.md +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/benchmarks/canary/__init__.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/benchmarks/canary/suite.v1.held-out.yaml +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/benchmarks/canary/suite.v1.public.yaml +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/chat_demo.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/chat_runtime.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/chat_slash.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/ci/__init__.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/cli.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/cloud/__init__.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/__init__.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/_check_helpers.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/_init_demo_agent.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/_init_tests.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/_init_wizard.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/add_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/assertion_wizard.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/autopr_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/baseline_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/benchmark_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/capture_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/chat_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/ci_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/demo_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/diff_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/expand_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/feedback_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/fleet_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/freshness_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/generate_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/golden_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/gym_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/hooks_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/import_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/init_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/judge_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/log_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/mcp_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/model_check_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/model_check_render.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/openclaw_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/progress_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/quarantine_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/replay_trace_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/run/__init__.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/run/_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/run/_executor.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/run/_helpers.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/run/_runner.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/simulate_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/since_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/skill_agent_runner.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/skill_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/slack_digest_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/telemetry_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/traces_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/validate_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/commands/visual_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/compare.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/__init__.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/behavioral_anomalies.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/benchmark_hardening.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/budget.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/canary_suite.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/cassette.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/celebrations.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/contract_diff.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/dashboard.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/discord_notifier.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/drift_classifier.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/drift_kind.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/eval_profiles.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/judge_cache.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/llm_configs.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/llm_provider.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/mcp_contract.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/messages.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/model_check_scoring.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/model_provider_runner.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/model_runtime_detector.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/model_snapshots.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/observability.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/otel_semconv.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/parallel.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/project_state.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/quarantine.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/rationale.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/regression_synth.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/replay_pipeline.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/retry.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/root_cause_hint.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/security.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/semantic_diff.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/simulation.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/slack_notifier.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/test_quality.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/tool_categories.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/tracing.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/turn_coherence.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/variant_clusterer.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/core/watcher.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/evaluators/__init__.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/evaluators/cost_evaluator.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/evaluators/latency_evaluator.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/evaluators/pii_evaluator.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/evaluators/safety_evaluator.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/evaluators/statistical_evaluator.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/expander.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/exporters/__init__.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/exporters/html_exporter.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/importers/__init__.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/importers/log_importer.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/recorder.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/reporters/__init__.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/reporters/_templates.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/reporters/json_reporter.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/reporters/trace_live_reporter.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/reporters/trace_reporter.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/__init__.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/adapters/base.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/adapters/claude_code_adapter.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/adapters/codex_adapter.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/adapters/crewai_adapter.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/adapters/custom_adapter.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/adapters/langgraph_adapter.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/adapters/openclaw_adapter.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/agent_runner.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/agent_types.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/constants.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/evaluators/__init__.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/evaluators/_security_checks.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/evaluators/_security_patterns.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/evaluators/deterministic.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/evaluators/orchestrator.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/evaluators/rubric.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/parser.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/runner.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/security_scanner.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/test_generator.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/test_helpers.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/types.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/ui_utils.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/skills/validator.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/storage/__init__.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/storage/database.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/telemetry/__init__.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/telemetry/client.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/telemetry/config.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/telemetry/decorators.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/telemetry/events.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/templates/patterns/code-generation.yaml +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/templates/patterns/cost-budget.yaml +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/templates/patterns/customer-support.yaml +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/templates/patterns/data-analysis.yaml +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/templates/patterns/latency-budget.yaml +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/templates/patterns/multi-turn-memory.yaml +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/templates/patterns/output-format.yaml +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/templates/patterns/rag-citation.yaml +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/templates/patterns/rag-grounding.yaml +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/templates/patterns/research-synthesis.yaml +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/templates/patterns/safety-refusal.yaml +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/templates/patterns/tool-error-handling.yaml +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/templates/patterns/tool-not-called.yaml +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/templates/patterns/tool-sequence.yaml +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/templates/patterns/wrong-tool-chosen.yaml +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/test_generation.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/test_generation_constants.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/test_generation_discovery.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/test_generation_seeds.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/test_generation_synthesis.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/test_generation_types.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/trace_cmd/__init__.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/trace_cmd/collector.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/trace_cmd/patcher.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/trace_cmd/runner.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/tracking/database.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/version_check.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/visualization/__init__.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/visualization/_mermaid.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview/visualization/_template.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview.egg-info/dependency_links.txt +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview.egg-info/entry_points.txt +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/evalview.egg-info/top_level.txt +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/setup.cfg +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_adapters.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_autopr.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_behavioral_anomalies.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_benchmark_hardening.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_canary_suite.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_cassette.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_celebrations.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_chaos.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_ci_generate_comment.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_config_models.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_diff_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_diff_engine.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_dogfood_e2e.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_drift_classifier.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_e2e_snapshot_check.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_endpoint_detection.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_evaluator_accuracy.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_evaluator_orchestrator.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_fleet.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_forbidden_tools.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_freshness.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_generate_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_goal_drift.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_golden_store.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_healing.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_init_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_judge_cache.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_llm_provider.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_log_importer_csv.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_main_evaluator.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_mcp_contracts.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_messages.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_model_check_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_model_check_scoring.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_model_runtime_detector.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_model_snapshots.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_multi_turn_evaluation.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_new_features.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_noise_tracker.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_observability_regressions.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_otel_semconv.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_pii_evaluator.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_project_state.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_pytest_plugin.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_rationale.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_replay_pipeline.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_retrieval_lineage.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_root_cause.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_root_cause_hint.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_run_cmd_tags.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_run_guidance.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_runner.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_security.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_semantic_diff.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_simulation.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_snapshot_generated_workflow.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_snapshot_json_output.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_statistical_evaluator.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_telemetry_client.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_turn_coherence.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_types.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_validate_cmd.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_verdict.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_version_check.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_visualization_generators.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_week1_integration.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_week2_quarantine.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_week2_review_fixes.py +0 -0
- {evalview-0.8.0 → evalview-0.9.0}/tests/test_week2_since_and_comment.py +0 -0
evalview-0.9.0/PKG-INFO
ADDED
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: evalview
|
|
3
|
+
Version: 0.9.0
|
|
4
|
+
Summary: Open-source testing and regression detection framework for AI agents. Golden baseline diffing, CI/CD integration, works with LangGraph, CrewAI, OpenAI, Anthropic Claude, HuggingFace, Ollama, and MCP.
|
|
5
|
+
Author-email: EvalView Team <hidai@evalview.com>
|
|
6
|
+
License-Expression: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://evalview.com
|
|
8
|
+
Project-URL: Documentation, https://github.com/hidai25/eval-view#readme
|
|
9
|
+
Project-URL: Repository, https://github.com/hidai25/eval-view.git
|
|
10
|
+
Project-URL: Issue Tracker, https://github.com/hidai25/eval-view/issues
|
|
11
|
+
Project-URL: Changelog, https://github.com/hidai25/eval-view/blob/main/CHANGELOG.md
|
|
12
|
+
Keywords: ai,agents,testing,evaluation,llm,langchain,langgraph,crewai,openai,anthropic,claude,huggingface,ollama,mcp,multi-agent,pytest-ai,ai-agent-testing,llm-testing,agent-evaluation,regression-testing,golden-baseline,ci-cd-testing,yaml-testing,tool-calling,agent-regression,llm-regression,skill-testing,mcp-contract-testing,pass-at-k,non-deterministic-testing,ai-ci-cd
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Topic :: Software Development :: Testing
|
|
21
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
22
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
23
|
+
Requires-Python: >=3.9
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
License-File: LICENSE
|
|
26
|
+
License-File: NOTICE
|
|
27
|
+
Requires-Dist: click>=8.1.0
|
|
28
|
+
Requires-Dist: pydantic>=2.5.0
|
|
29
|
+
Requires-Dist: pyyaml>=6.0
|
|
30
|
+
Requires-Dist: openai>=1.101.0
|
|
31
|
+
Requires-Dist: anthropic>=0.39.0
|
|
32
|
+
Requires-Dist: rich>=13.7.0
|
|
33
|
+
Requires-Dist: prompt_toolkit>=3.0.0
|
|
34
|
+
Requires-Dist: httpx>=0.26.0
|
|
35
|
+
Requires-Dist: python-dateutil>=2.8.2
|
|
36
|
+
Requires-Dist: python-dotenv>=1.0.0
|
|
37
|
+
Requires-Dist: jinja2>=3.0
|
|
38
|
+
Requires-Dist: jsonschema>=4.0.0
|
|
39
|
+
Requires-Dist: llm-prices>=1.4.0
|
|
40
|
+
Requires-Dist: tomli>=2.0; python_version < "3.11"
|
|
41
|
+
Provides-Extra: reports
|
|
42
|
+
Requires-Dist: plotly>=5.0; extra == "reports"
|
|
43
|
+
Provides-Extra: watch
|
|
44
|
+
Requires-Dist: watchdog>=3.0; extra == "watch"
|
|
45
|
+
Provides-Extra: schedule
|
|
46
|
+
Requires-Dist: croniter>=2.0; extra == "schedule"
|
|
47
|
+
Provides-Extra: telemetry
|
|
48
|
+
Requires-Dist: posthog>=3.0.0; extra == "telemetry"
|
|
49
|
+
Provides-Extra: cohere
|
|
50
|
+
Requires-Dist: cohere>=5.0.0; extra == "cohere"
|
|
51
|
+
Provides-Extra: mistral
|
|
52
|
+
Requires-Dist: mistralai>=1.0.0; extra == "mistral"
|
|
53
|
+
Provides-Extra: pydantic-ai
|
|
54
|
+
Requires-Dist: pydantic-ai-slim>=0.0.30; python_version >= "3.10" and extra == "pydantic-ai"
|
|
55
|
+
Provides-Extra: all
|
|
56
|
+
Requires-Dist: plotly>=5.0; extra == "all"
|
|
57
|
+
Requires-Dist: watchdog>=3.0; extra == "all"
|
|
58
|
+
Requires-Dist: croniter>=2.0; extra == "all"
|
|
59
|
+
Requires-Dist: posthog>=3.0.0; extra == "all"
|
|
60
|
+
Requires-Dist: cohere>=5.0.0; extra == "all"
|
|
61
|
+
Requires-Dist: mistralai>=1.0.0; extra == "all"
|
|
62
|
+
Requires-Dist: pydantic-ai-slim>=0.0.30; python_version >= "3.10" and extra == "all"
|
|
63
|
+
Provides-Extra: dev
|
|
64
|
+
Requires-Dist: pytest>=7.4.0; extra == "dev"
|
|
65
|
+
Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
|
|
66
|
+
Requires-Dist: pytest-cov>=4.1.0; extra == "dev"
|
|
67
|
+
Requires-Dist: black==24.10.0; extra == "dev"
|
|
68
|
+
Requires-Dist: mypy>=1.7.0; extra == "dev"
|
|
69
|
+
Requires-Dist: types-PyYAML>=6.0; extra == "dev"
|
|
70
|
+
Requires-Dist: ruff>=0.1.0; extra == "dev"
|
|
71
|
+
Requires-Dist: jinja2>=3.0; extra == "dev"
|
|
72
|
+
Requires-Dist: plotly>=5.0; extra == "dev"
|
|
73
|
+
Requires-Dist: watchdog>=3.0; extra == "dev"
|
|
74
|
+
Requires-Dist: anthropic>=0.39.0; extra == "dev"
|
|
75
|
+
Requires-Dist: fastapi>=0.109.0; extra == "dev"
|
|
76
|
+
Requires-Dist: uvicorn>=0.27.0; extra == "dev"
|
|
77
|
+
Requires-Dist: pydantic-ai-slim>=0.0.30; python_version >= "3.10" and extra == "dev"
|
|
78
|
+
Dynamic: license-file
|
|
79
|
+
|
|
80
|
+
<!-- mcp-name: io.github.hidai25/evalview-mcp -->
|
|
81
|
+
<!-- keywords: AI agent testing, regression detection, golden baselines -->
|
|
82
|
+
|
|
83
|
+
<p align="center">
|
|
84
|
+
<img src="assets/logo.png" alt="EvalView" width="350">
|
|
85
|
+
<br>
|
|
86
|
+
<strong>Snapshot testing for AI agents.</strong><br>
|
|
87
|
+
Record what your agent does today. Get told when it silently changes.
|
|
88
|
+
</p>
|
|
89
|
+
|
|
90
|
+
<p align="center">
|
|
91
|
+
<a href="https://pypi.org/project/evalview/"><img src="https://img.shields.io/pypi/v/evalview.svg?label=release" alt="PyPI version"></a>
|
|
92
|
+
<a href="https://pypi.org/project/evalview/"><img src="https://img.shields.io/pypi/dm/evalview.svg?label=downloads" alt="PyPI downloads"></a>
|
|
93
|
+
<a href="https://github.com/hidai25/eval-view/actions/workflows/ci.yml"><img src="https://github.com/hidai25/eval-view/actions/workflows/ci.yml/badge.svg?branch=main" alt="Package CI"></a>
|
|
94
|
+
<a href="https://github.com/hidai25/eval-view/actions/workflows/dogfood.yml"><img src="https://github.com/hidai25/eval-view/actions/workflows/dogfood.yml/badge.svg?branch=main" alt="Core Dogfood"></a>
|
|
95
|
+
<a href="https://github.com/hidai25/eval-view/actions/workflows/dogfood-live.yml"><img src="https://github.com/hidai25/eval-view/actions/workflows/dogfood-live.yml/badge.svg?branch=main&event=workflow_dispatch" alt="Live Provider Checks (manual)"></a>
|
|
96
|
+
<a href="https://github.com/hidai25/eval-view/stargazers"><img src="https://img.shields.io/github/stars/hidai25/eval-view?style=social" alt="GitHub stars"></a>
|
|
97
|
+
<a href="https://opensource.org/licenses/Apache-2.0"><img src="https://img.shields.io/badge/License-Apache_2.0-blue.svg" alt="License"></a>
|
|
98
|
+
</p>
|
|
99
|
+
|
|
100
|
+
---
|
|
101
|
+
|
|
102
|
+
Your agent returns `200` and looks fine. But a model update, a provider change, or a one-line prompt edit just made it skip a clarification, call the wrong tool, or quietly drop output quality. Your tests still pass. Your users notice before you do.
|
|
103
|
+
|
|
104
|
+
**EvalView snapshots your agent's behavior — the tools it calls, in what order, with what output — and tells you the moment that behavior changes.** Like Jest snapshots, but for tool-calling, multi-turn agents.
|
|
105
|
+
|
|
106
|
+
[](https://github.com/user-attachments/assets/96d8b5f7-3561-44a1-86a4-270fb0d1d8a6)
|
|
107
|
+
|
|
108
|
+
<sub>↑ 30-second live demo — no API key needed</sub>
|
|
109
|
+
|
|
110
|
+
## Quick Start
|
|
111
|
+
|
|
112
|
+
> **OpenAI adapter migration:** OpenAI shut down the Assistants API on **August 26, 2026**.
|
|
113
|
+
> EvalView 0.9.0 uses the Responses API. If you use `openai-assistants`, migrate your
|
|
114
|
+
> model, instructions and tools or configure a dashboard Prompt before running checks.
|
|
115
|
+
> [OpenAI migration guide](docs/OPENAI_MIGRATION.md)
|
|
116
|
+
|
|
117
|
+
```bash
|
|
118
|
+
pip install evalview
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
```bash
|
|
122
|
+
evalview snapshot # Record your agent's current behavior as the baseline
|
|
123
|
+
evalview check # After any change, diff against the baseline
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
That's the whole loop. `check` returns one of:
|
|
127
|
+
|
|
128
|
+
```
|
|
129
|
+
✓ login-flow PASSED behavior matches baseline
|
|
130
|
+
⚠ refund-request TOOLS_CHANGED called a different tool, or in a different order
|
|
131
|
+
✗ billing-dispute REGRESSION score dropped — output quality fell
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
It diffs the **whole trajectory** — tool names, parameters, and order — not just the final string. The deterministic tool + sequence diff runs offline, with no API key. Add an LLM judge only when you want output-quality scoring.
|
|
135
|
+
|
|
136
|
+
Executing your agent can still incur backend API charges: `--no-judge` skips the
|
|
137
|
+
judge, not those calls. Embedding-based semantic comparison is opt-in.
|
|
138
|
+
|
|
139
|
+
No agent yet? See it work in 30 seconds:
|
|
140
|
+
|
|
141
|
+
```bash
|
|
142
|
+
evalview demo
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
## Why snapshot testing (and not assertions)?
|
|
146
|
+
|
|
147
|
+
Most eval tools ask you to *write down what "good" looks like* — assertions, metrics, rubrics. That's a lot of upfront work, and you can only catch the failures you thought to assert.
|
|
148
|
+
|
|
149
|
+
EvalView inverts it: **it records what your agent actually does now, and flags any drift from that.** You catch regressions you never anticipated, with zero assertions written. When the new behavior is correct, `evalview snapshot` accepts it as the new baseline — same as updating a snapshot in Jest.
|
|
150
|
+
|
|
151
|
+
| | EvalView | Assertion-based eval tools |
|
|
152
|
+
|---|---|---|
|
|
153
|
+
| Setup | Record current behavior | Write assertions/metrics first |
|
|
154
|
+
| Catches | Any drift from baseline | Only what you asserted |
|
|
155
|
+
| Non-determinism | Multi-variant baselines (up to 5 valid paths) | You handle it |
|
|
156
|
+
| Unit of comparison | Full tool-call trajectory | Usually final output |
|
|
157
|
+
|
|
158
|
+
This makes EvalView a **merge-time regression gate**, which is a different job from observability (Langfuse, LangSmith) or metric scoring (promptfoo, DeepEval, Braintrust). Many teams run one of those for visibility **and** EvalView as the gate. [Honest comparisons →](docs/COMPARISONS.md)
|
|
159
|
+
|
|
160
|
+
## EvalView tests itself in public, every day
|
|
161
|
+
|
|
162
|
+
Every day at 09:00 UTC, on pull requests, and on pushes to main,
|
|
163
|
+
[Core Dogfood](.github/workflows/dogfood.yml) exercises the non-live test suite,
|
|
164
|
+
type checks, local mock-agent `snapshot` / `check`, `evalview demo`, end-to-end
|
|
165
|
+
flows, and an `evalview monitor` smoke test. It uses no paid API credentials and
|
|
166
|
+
makes no paid inference calls. GitHub runner usage is separate.
|
|
167
|
+
|
|
168
|
+
[Live Provider Checks](.github/workflows/dogfood-live.yml) test the real evaluator and
|
|
169
|
+
chat assistant only when a maintainer explicitly opts into paid API use on main.
|
|
170
|
+
They have no automatic schedule. Their badge records the last manual run; a green
|
|
171
|
+
core badge does not establish live-provider health or rule out provider drift.
|
|
172
|
+
|
|
173
|
+
Package CI, core dogfood, and live checks have separate badges. Failed or incomplete
|
|
174
|
+
checks remain visible within their scope, with logs and reports preserved as
|
|
175
|
+
artifacts. Rolling issues use separate `dogfood-core` and `dogfood-live` labels.
|
|
176
|
+
A provider outage, exhausted quota, or missing credential means live health is
|
|
177
|
+
unavailable; it does not prove an agent regression.
|
|
178
|
+
|
|
179
|
+
The historical [incident #264](https://github.com/hidai25/eval-view/issues/264)
|
|
180
|
+
remains available for maintainer review of fresh evidence from both scopes. Neither
|
|
181
|
+
workflow automatically closes it. Trust warnings are evidence to investigate,
|
|
182
|
+
not proof of gaming or of a particular root cause.
|
|
183
|
+
|
|
184
|
+
[Core runs →](https://github.com/hidai25/eval-view/actions/workflows/dogfood.yml) · [Manual live runs →](https://github.com/hidai25/eval-view/actions/workflows/dogfood-live.yml) · [Run and triage guide →](docs/INTERNAL_DOGFOODING.md#daily-failure-triage)
|
|
185
|
+
|
|
186
|
+
## CI: block regressions in every PR
|
|
187
|
+
|
|
188
|
+
```yaml
|
|
189
|
+
# .github/workflows/evalview.yml
|
|
190
|
+
name: EvalView
|
|
191
|
+
on: [pull_request]
|
|
192
|
+
jobs:
|
|
193
|
+
agent-check:
|
|
194
|
+
runs-on: ubuntu-latest
|
|
195
|
+
permissions: { pull-requests: write }
|
|
196
|
+
steps:
|
|
197
|
+
- uses: actions/checkout@v4
|
|
198
|
+
- uses: hidai25/eval-view@v0.9.0
|
|
199
|
+
with:
|
|
200
|
+
openai-api-key: ${{ secrets.OPENAI_API_KEY }}
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
You get a PR comment with the diff, cost/latency deltas, and a pass/fail gate. [CI/CD guide →](docs/CI_CD.md)
|
|
204
|
+
|
|
205
|
+
## Works with your stack
|
|
206
|
+
|
|
207
|
+
LangGraph · CrewAI · OpenAI · Claude · Mistral · Ollama · MCP · **any HTTP API**.
|
|
208
|
+
|
|
209
|
+
```bash
|
|
210
|
+
evalview check --agent http://localhost:8000/invoke
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
[Framework details →](docs/FRAMEWORK_SUPPORT.md)
|
|
214
|
+
|
|
215
|
+
## Use it as a library
|
|
216
|
+
|
|
217
|
+
```python
|
|
218
|
+
from evalview import gate
|
|
219
|
+
|
|
220
|
+
result = gate(test_dir="tests/")
|
|
221
|
+
result.passed # bool
|
|
222
|
+
result.diffs # per-test scores and tool diffs
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
[Python API →](docs/CLI_REFERENCE.md#python-api)
|
|
226
|
+
|
|
227
|
+
## More
|
|
228
|
+
|
|
229
|
+
EvalView also does multi-turn testing, statistical/pass@k runs, record/replay cassettes, model-drift canaries, production monitoring with Slack alerts, and auto-generated regression tests from incidents. These are power-user features — start with `snapshot` and `check`, reach for the rest when you need them.
|
|
230
|
+
|
|
231
|
+
→ [Full feature reference](docs/CLI_REFERENCE.md) · [Getting Started](docs/GETTING_STARTED.md) · [FAQ](docs/FAQ.md)
|
|
232
|
+
|
|
233
|
+
→ [Documentation index](docs/README.md) · [OpenAI migration](docs/OPENAI_MIGRATION.md) · [Release process](docs/RELEASING.md)
|
|
234
|
+
|
|
235
|
+
### Why I built EvalView
|
|
236
|
+
|
|
237
|
+
An agent that looked successful kept pulling entire documents into its context and made one question cost $42.93. That experience led me to build EvalView. I wrote about it in [“I Was Running an AI Casino. Then I Started Writing Tests for My Agents”](https://medium.com/@hidaibarmor/i-was-running-an-ai-casino-then-i-started-writing-tests-for-my-agents-93cb3468ce1e). The December 2025 post is the origin story; use the current docs for setup and commands.
|
|
238
|
+
|
|
239
|
+
## Contributing
|
|
240
|
+
|
|
241
|
+
This is a young project built mostly by one developer. Issues, PRs, and "I tried it and X was confusing" feedback are all genuinely valuable.
|
|
242
|
+
|
|
243
|
+
- [Open an issue](https://github.com/hidai25/eval-view/issues) · [Discussions](https://github.com/hidai25/eval-view/discussions) · [CONTRIBUTING.md](CONTRIBUTING.md)
|
|
244
|
+
|
|
245
|
+
**License:** Apache 2.0
|
|
246
|
+
|
|
247
|
+
---
|
|
248
|
+
|
|
249
|
+
[](https://star-history.com/#hidai25/eval-view&Date)
|
evalview-0.9.0/README.md
ADDED
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
<!-- mcp-name: io.github.hidai25/evalview-mcp -->
|
|
2
|
+
<!-- keywords: AI agent testing, regression detection, golden baselines -->
|
|
3
|
+
|
|
4
|
+
<p align="center">
|
|
5
|
+
<img src="assets/logo.png" alt="EvalView" width="350">
|
|
6
|
+
<br>
|
|
7
|
+
<strong>Snapshot testing for AI agents.</strong><br>
|
|
8
|
+
Record what your agent does today. Get told when it silently changes.
|
|
9
|
+
</p>
|
|
10
|
+
|
|
11
|
+
<p align="center">
|
|
12
|
+
<a href="https://pypi.org/project/evalview/"><img src="https://img.shields.io/pypi/v/evalview.svg?label=release" alt="PyPI version"></a>
|
|
13
|
+
<a href="https://pypi.org/project/evalview/"><img src="https://img.shields.io/pypi/dm/evalview.svg?label=downloads" alt="PyPI downloads"></a>
|
|
14
|
+
<a href="https://github.com/hidai25/eval-view/actions/workflows/ci.yml"><img src="https://github.com/hidai25/eval-view/actions/workflows/ci.yml/badge.svg?branch=main" alt="Package CI"></a>
|
|
15
|
+
<a href="https://github.com/hidai25/eval-view/actions/workflows/dogfood.yml"><img src="https://github.com/hidai25/eval-view/actions/workflows/dogfood.yml/badge.svg?branch=main" alt="Core Dogfood"></a>
|
|
16
|
+
<a href="https://github.com/hidai25/eval-view/actions/workflows/dogfood-live.yml"><img src="https://github.com/hidai25/eval-view/actions/workflows/dogfood-live.yml/badge.svg?branch=main&event=workflow_dispatch" alt="Live Provider Checks (manual)"></a>
|
|
17
|
+
<a href="https://github.com/hidai25/eval-view/stargazers"><img src="https://img.shields.io/github/stars/hidai25/eval-view?style=social" alt="GitHub stars"></a>
|
|
18
|
+
<a href="https://opensource.org/licenses/Apache-2.0"><img src="https://img.shields.io/badge/License-Apache_2.0-blue.svg" alt="License"></a>
|
|
19
|
+
</p>
|
|
20
|
+
|
|
21
|
+
---
|
|
22
|
+
|
|
23
|
+
Your agent returns `200` and looks fine. But a model update, a provider change, or a one-line prompt edit just made it skip a clarification, call the wrong tool, or quietly drop output quality. Your tests still pass. Your users notice before you do.
|
|
24
|
+
|
|
25
|
+
**EvalView snapshots your agent's behavior — the tools it calls, in what order, with what output — and tells you the moment that behavior changes.** Like Jest snapshots, but for tool-calling, multi-turn agents.
|
|
26
|
+
|
|
27
|
+
[](https://github.com/user-attachments/assets/96d8b5f7-3561-44a1-86a4-270fb0d1d8a6)
|
|
28
|
+
|
|
29
|
+
<sub>↑ 30-second live demo — no API key needed</sub>
|
|
30
|
+
|
|
31
|
+
## Quick Start
|
|
32
|
+
|
|
33
|
+
> **OpenAI adapter migration:** OpenAI shut down the Assistants API on **August 26, 2026**.
|
|
34
|
+
> EvalView 0.9.0 uses the Responses API. If you use `openai-assistants`, migrate your
|
|
35
|
+
> model, instructions and tools or configure a dashboard Prompt before running checks.
|
|
36
|
+
> [OpenAI migration guide](docs/OPENAI_MIGRATION.md)
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
pip install evalview
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
evalview snapshot # Record your agent's current behavior as the baseline
|
|
44
|
+
evalview check # After any change, diff against the baseline
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
That's the whole loop. `check` returns one of:
|
|
48
|
+
|
|
49
|
+
```
|
|
50
|
+
✓ login-flow PASSED behavior matches baseline
|
|
51
|
+
⚠ refund-request TOOLS_CHANGED called a different tool, or in a different order
|
|
52
|
+
✗ billing-dispute REGRESSION score dropped — output quality fell
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
It diffs the **whole trajectory** — tool names, parameters, and order — not just the final string. The deterministic tool + sequence diff runs offline, with no API key. Add an LLM judge only when you want output-quality scoring.
|
|
56
|
+
|
|
57
|
+
Executing your agent can still incur backend API charges: `--no-judge` skips the
|
|
58
|
+
judge, not those calls. Embedding-based semantic comparison is opt-in.
|
|
59
|
+
|
|
60
|
+
No agent yet? See it work in 30 seconds:
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
evalview demo
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
## Why snapshot testing (and not assertions)?
|
|
67
|
+
|
|
68
|
+
Most eval tools ask you to *write down what "good" looks like* — assertions, metrics, rubrics. That's a lot of upfront work, and you can only catch the failures you thought to assert.
|
|
69
|
+
|
|
70
|
+
EvalView inverts it: **it records what your agent actually does now, and flags any drift from that.** You catch regressions you never anticipated, with zero assertions written. When the new behavior is correct, `evalview snapshot` accepts it as the new baseline — same as updating a snapshot in Jest.
|
|
71
|
+
|
|
72
|
+
| | EvalView | Assertion-based eval tools |
|
|
73
|
+
|---|---|---|
|
|
74
|
+
| Setup | Record current behavior | Write assertions/metrics first |
|
|
75
|
+
| Catches | Any drift from baseline | Only what you asserted |
|
|
76
|
+
| Non-determinism | Multi-variant baselines (up to 5 valid paths) | You handle it |
|
|
77
|
+
| Unit of comparison | Full tool-call trajectory | Usually final output |
|
|
78
|
+
|
|
79
|
+
This makes EvalView a **merge-time regression gate**, which is a different job from observability (Langfuse, LangSmith) or metric scoring (promptfoo, DeepEval, Braintrust). Many teams run one of those for visibility **and** EvalView as the gate. [Honest comparisons →](docs/COMPARISONS.md)
|
|
80
|
+
|
|
81
|
+
## EvalView tests itself in public, every day
|
|
82
|
+
|
|
83
|
+
Every day at 09:00 UTC, on pull requests, and on pushes to main,
|
|
84
|
+
[Core Dogfood](.github/workflows/dogfood.yml) exercises the non-live test suite,
|
|
85
|
+
type checks, local mock-agent `snapshot` / `check`, `evalview demo`, end-to-end
|
|
86
|
+
flows, and an `evalview monitor` smoke test. It uses no paid API credentials and
|
|
87
|
+
makes no paid inference calls. GitHub runner usage is separate.
|
|
88
|
+
|
|
89
|
+
[Live Provider Checks](.github/workflows/dogfood-live.yml) test the real evaluator and
|
|
90
|
+
chat assistant only when a maintainer explicitly opts into paid API use on main.
|
|
91
|
+
They have no automatic schedule. Their badge records the last manual run; a green
|
|
92
|
+
core badge does not establish live-provider health or rule out provider drift.
|
|
93
|
+
|
|
94
|
+
Package CI, core dogfood, and live checks have separate badges. Failed or incomplete
|
|
95
|
+
checks remain visible within their scope, with logs and reports preserved as
|
|
96
|
+
artifacts. Rolling issues use separate `dogfood-core` and `dogfood-live` labels.
|
|
97
|
+
A provider outage, exhausted quota, or missing credential means live health is
|
|
98
|
+
unavailable; it does not prove an agent regression.
|
|
99
|
+
|
|
100
|
+
The historical [incident #264](https://github.com/hidai25/eval-view/issues/264)
|
|
101
|
+
remains available for maintainer review of fresh evidence from both scopes. Neither
|
|
102
|
+
workflow automatically closes it. Trust warnings are evidence to investigate,
|
|
103
|
+
not proof of gaming or of a particular root cause.
|
|
104
|
+
|
|
105
|
+
[Core runs →](https://github.com/hidai25/eval-view/actions/workflows/dogfood.yml) · [Manual live runs →](https://github.com/hidai25/eval-view/actions/workflows/dogfood-live.yml) · [Run and triage guide →](docs/INTERNAL_DOGFOODING.md#daily-failure-triage)
|
|
106
|
+
|
|
107
|
+
## CI: block regressions in every PR
|
|
108
|
+
|
|
109
|
+
```yaml
|
|
110
|
+
# .github/workflows/evalview.yml
|
|
111
|
+
name: EvalView
|
|
112
|
+
on: [pull_request]
|
|
113
|
+
jobs:
|
|
114
|
+
agent-check:
|
|
115
|
+
runs-on: ubuntu-latest
|
|
116
|
+
permissions: { pull-requests: write }
|
|
117
|
+
steps:
|
|
118
|
+
- uses: actions/checkout@v4
|
|
119
|
+
- uses: hidai25/eval-view@v0.9.0
|
|
120
|
+
with:
|
|
121
|
+
openai-api-key: ${{ secrets.OPENAI_API_KEY }}
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
You get a PR comment with the diff, cost/latency deltas, and a pass/fail gate. [CI/CD guide →](docs/CI_CD.md)
|
|
125
|
+
|
|
126
|
+
## Works with your stack
|
|
127
|
+
|
|
128
|
+
LangGraph · CrewAI · OpenAI · Claude · Mistral · Ollama · MCP · **any HTTP API**.
|
|
129
|
+
|
|
130
|
+
```bash
|
|
131
|
+
evalview check --agent http://localhost:8000/invoke
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
[Framework details →](docs/FRAMEWORK_SUPPORT.md)
|
|
135
|
+
|
|
136
|
+
## Use it as a library
|
|
137
|
+
|
|
138
|
+
```python
|
|
139
|
+
from evalview import gate
|
|
140
|
+
|
|
141
|
+
result = gate(test_dir="tests/")
|
|
142
|
+
result.passed # bool
|
|
143
|
+
result.diffs # per-test scores and tool diffs
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
[Python API →](docs/CLI_REFERENCE.md#python-api)
|
|
147
|
+
|
|
148
|
+
## More
|
|
149
|
+
|
|
150
|
+
EvalView also does multi-turn testing, statistical/pass@k runs, record/replay cassettes, model-drift canaries, production monitoring with Slack alerts, and auto-generated regression tests from incidents. These are power-user features — start with `snapshot` and `check`, reach for the rest when you need them.
|
|
151
|
+
|
|
152
|
+
→ [Full feature reference](docs/CLI_REFERENCE.md) · [Getting Started](docs/GETTING_STARTED.md) · [FAQ](docs/FAQ.md)
|
|
153
|
+
|
|
154
|
+
→ [Documentation index](docs/README.md) · [OpenAI migration](docs/OPENAI_MIGRATION.md) · [Release process](docs/RELEASING.md)
|
|
155
|
+
|
|
156
|
+
### Why I built EvalView
|
|
157
|
+
|
|
158
|
+
An agent that looked successful kept pulling entire documents into its context and made one question cost $42.93. That experience led me to build EvalView. I wrote about it in [“I Was Running an AI Casino. Then I Started Writing Tests for My Agents”](https://medium.com/@hidaibarmor/i-was-running-an-ai-casino-then-i-started-writing-tests-for-my-agents-93cb3468ce1e). The December 2025 post is the origin story; use the current docs for setup and commands.
|
|
159
|
+
|
|
160
|
+
## Contributing
|
|
161
|
+
|
|
162
|
+
This is a young project built mostly by one developer. Issues, PRs, and "I tried it and X was confusing" feedback are all genuinely valuable.
|
|
163
|
+
|
|
164
|
+
- [Open an issue](https://github.com/hidai25/eval-view/issues) · [Discussions](https://github.com/hidai25/eval-view/discussions) · [CONTRIBUTING.md](CONTRIBUTING.md)
|
|
165
|
+
|
|
166
|
+
**License:** Apache 2.0
|
|
167
|
+
|
|
168
|
+
---
|
|
169
|
+
|
|
170
|
+
[](https://star-history.com/#hidai25/eval-view&Date)
|
|
@@ -49,17 +49,26 @@ class CohereAdapter(AgentAdapter):
|
|
|
49
49
|
async with tracer.start_span_async("Cohere Agent", SpanKind.AGENT):
|
|
50
50
|
api_start = datetime.now()
|
|
51
51
|
|
|
52
|
-
# Send request to Cohere
|
|
52
|
+
# Send request to Cohere. The import is safe here: __init__ already
|
|
53
|
+
# verified the optional cohere package is installed.
|
|
54
|
+
from cohere import UserChatMessageV2
|
|
55
|
+
|
|
53
56
|
response = await self.client.chat(
|
|
54
57
|
model=self.model,
|
|
55
|
-
messages=[
|
|
58
|
+
messages=[UserChatMessageV2(content=query)]
|
|
56
59
|
)
|
|
57
|
-
|
|
60
|
+
|
|
58
61
|
api_end = datetime.now()
|
|
59
62
|
api_latency = (api_end - api_start).total_seconds() * 1000
|
|
60
63
|
|
|
61
|
-
# Extract AI's response text
|
|
62
|
-
|
|
64
|
+
# Extract the AI's response text. content is Optional, and items may
|
|
65
|
+
# be thinking blocks that carry no text payload.
|
|
66
|
+
final_answer = ""
|
|
67
|
+
for item in response.message.content or []:
|
|
68
|
+
text = getattr(item, "text", None)
|
|
69
|
+
if text:
|
|
70
|
+
final_answer = text
|
|
71
|
+
break
|
|
63
72
|
|
|
64
73
|
# 2. Extract Token Usage (Changes for compatibility with V1 and V2 SDKs)
|
|
65
74
|
input_tokens = 0
|
|
@@ -262,8 +262,11 @@ class CrewAINativeAdapter(AgentAdapter):
|
|
|
262
262
|
|
|
263
263
|
rationale = RationaleCollector()
|
|
264
264
|
last_agent: Optional[str] = None
|
|
265
|
-
|
|
266
|
-
|
|
265
|
+
calls_by_step: List[Optional[Dict[str, Any]]] = (
|
|
266
|
+
list(tool_calls) if tool_calls else [None] * len(steps)
|
|
267
|
+
)
|
|
268
|
+
for i, (st, call) in enumerate(zip(steps, calls_by_step)):
|
|
269
|
+
agent_name = call.get("agent") if call else None
|
|
267
270
|
if agent_name and agent_name != last_agent:
|
|
268
271
|
rationale.capture_branch(
|
|
269
272
|
step_id=st.step_id,
|
|
@@ -317,9 +317,13 @@ class GooseAdapter(AgentAdapter):
|
|
|
317
317
|
if tokens_data:
|
|
318
318
|
if isinstance(tokens_data, dict):
|
|
319
319
|
total_tokens = TokenUsage(
|
|
320
|
-
input_tokens=
|
|
321
|
-
|
|
322
|
-
|
|
320
|
+
input_tokens=int(
|
|
321
|
+
tokens_data.get("input", tokens_data.get("input_tokens", 0)) or 0
|
|
322
|
+
),
|
|
323
|
+
output_tokens=int(
|
|
324
|
+
tokens_data.get("output", tokens_data.get("output_tokens", 0)) or 0
|
|
325
|
+
),
|
|
326
|
+
cached_tokens=int(tokens_data.get("cached", 0) or 0),
|
|
323
327
|
)
|
|
324
328
|
elif isinstance(tokens_data, int):
|
|
325
329
|
total_tokens = TokenUsage(output_tokens=tokens_data)
|
|
@@ -153,8 +153,12 @@ class HTTPAdapter(AgentAdapter):
|
|
|
153
153
|
input_tokens = 0
|
|
154
154
|
output_tokens = 0
|
|
155
155
|
if isinstance(tokens_data, dict):
|
|
156
|
-
input_tokens =
|
|
157
|
-
|
|
156
|
+
input_tokens = int(
|
|
157
|
+
tokens_data.get("input", tokens_data.get("input_tokens", 0)) or 0
|
|
158
|
+
)
|
|
159
|
+
output_tokens = int(
|
|
160
|
+
tokens_data.get("output", tokens_data.get("output_tokens", 0)) or 0
|
|
161
|
+
)
|
|
158
162
|
|
|
159
163
|
tracer.record_llm_call(
|
|
160
164
|
model=model_name,
|
|
@@ -250,9 +254,15 @@ class HTTPAdapter(AgentAdapter):
|
|
|
250
254
|
if isinstance(tokens_data, dict):
|
|
251
255
|
# Nested format: {"input": 100, "output": 500, "cached": 50}
|
|
252
256
|
total_tokens = TokenUsage(
|
|
253
|
-
input_tokens=
|
|
254
|
-
|
|
255
|
-
|
|
257
|
+
input_tokens=int(
|
|
258
|
+
tokens_data.get("input", tokens_data.get("input_tokens", 0)) or 0
|
|
259
|
+
),
|
|
260
|
+
output_tokens=int(
|
|
261
|
+
tokens_data.get("output", tokens_data.get("output_tokens", 0)) or 0
|
|
262
|
+
),
|
|
263
|
+
cached_tokens=int(
|
|
264
|
+
tokens_data.get("cached", tokens_data.get("cached_tokens", 0)) or 0
|
|
265
|
+
),
|
|
256
266
|
)
|
|
257
267
|
elif isinstance(tokens_data, int):
|
|
258
268
|
# Simple total: {"tokens": 1500}
|
|
@@ -128,6 +128,7 @@ class LangGraphAdapter(AgentAdapter):
|
|
|
128
128
|
|
|
129
129
|
# Step 2: Create run with streaming
|
|
130
130
|
input_key = context.get("input_key", self.input_key)
|
|
131
|
+
run_input: Dict[str, Any]
|
|
131
132
|
if input_key:
|
|
132
133
|
run_input = {input_key: query}
|
|
133
134
|
else:
|
|
@@ -559,7 +560,7 @@ class LangGraphAdapter(AgentAdapter):
|
|
|
559
560
|
return StepTrace(
|
|
560
561
|
step_id=step_data.get("id", f"step-{index}"),
|
|
561
562
|
step_name=step_data.get("name", f"Step {index + 1}"),
|
|
562
|
-
tool_name=step_data.get("tool"),
|
|
563
|
+
tool_name=step_data.get("tool") or "unknown",
|
|
563
564
|
parameters=step_data.get("parameters", {}),
|
|
564
565
|
output=step_data.get("output"),
|
|
565
566
|
success=step_data.get("success", True),
|
|
@@ -579,7 +580,7 @@ class LangGraphAdapter(AgentAdapter):
|
|
|
579
580
|
return StepTrace(
|
|
580
581
|
step_id=event.get("id", f"step-{datetime.now().timestamp()}"),
|
|
581
582
|
step_name=content[:50] if content else "Step",
|
|
582
|
-
tool_name=event.get("tool"),
|
|
583
|
+
tool_name=event.get("tool") or "unknown",
|
|
583
584
|
parameters=event.get("parameters", {}),
|
|
584
585
|
output=content,
|
|
585
586
|
success=True,
|
|
@@ -344,6 +344,8 @@ class MCPAdapter(AgentAdapter):
|
|
|
344
344
|
self, process: asyncio.subprocess.Process, method: str, params: Dict
|
|
345
345
|
) -> Dict:
|
|
346
346
|
"""Send JSON-RPC request via stdio."""
|
|
347
|
+
# Process is always spawned with stdin/stdout=PIPE
|
|
348
|
+
assert process.stdin is not None and process.stdout is not None
|
|
347
349
|
self._request_id += 1
|
|
348
350
|
request = {
|
|
349
351
|
"jsonrpc": "2.0",
|
|
@@ -377,6 +379,8 @@ class MCPAdapter(AgentAdapter):
|
|
|
377
379
|
self, process: asyncio.subprocess.Process, method: str, params: Dict
|
|
378
380
|
) -> None:
|
|
379
381
|
"""Send JSON-RPC notification (no response expected)."""
|
|
382
|
+
# Process is always spawned with stdin=PIPE
|
|
383
|
+
assert process.stdin is not None
|
|
380
384
|
notification = {
|
|
381
385
|
"jsonrpc": "2.0",
|
|
382
386
|
"method": method,
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import os
|
|
2
2
|
import logging
|
|
3
3
|
from datetime import datetime
|
|
4
|
-
from typing import Any, Optional, Dict
|
|
4
|
+
from typing import Any, List, Optional, Dict
|
|
5
5
|
|
|
6
6
|
try:
|
|
7
|
-
from mistralai import Mistral
|
|
7
|
+
from mistralai import Mistral, UserMessage
|
|
8
|
+
from mistralai.models import Messages
|
|
8
9
|
except ImportError:
|
|
9
10
|
raise ImportError(
|
|
10
11
|
"The mistralai package is required for the Mistral adapter. "
|
|
@@ -51,18 +52,28 @@ class MistralAdapter(AgentAdapter):
|
|
|
51
52
|
api_start = datetime.now()
|
|
52
53
|
|
|
53
54
|
# Send request to Mistral
|
|
55
|
+
messages: List[Messages] = [UserMessage(content=query)]
|
|
54
56
|
response = await self.client.chat.complete_async(
|
|
55
57
|
model=self.model,
|
|
56
|
-
messages=
|
|
58
|
+
messages=messages
|
|
57
59
|
)
|
|
58
|
-
|
|
60
|
+
|
|
59
61
|
api_end = datetime.now()
|
|
60
62
|
api_latency = (api_end - api_start).total_seconds() * 1000
|
|
61
63
|
|
|
62
|
-
# Extract AI's response text
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
64
|
+
# Extract the AI's response text. content may be a plain string or a
|
|
65
|
+
# list of chunks (text, think, image, ...); join the text-bearing ones.
|
|
66
|
+
content = response.choices[0].message.content
|
|
67
|
+
if isinstance(content, str):
|
|
68
|
+
final_answer = content
|
|
69
|
+
elif isinstance(content, list):
|
|
70
|
+
final_answer = "".join(getattr(chunk, "text", "") or "" for chunk in content)
|
|
71
|
+
else:
|
|
72
|
+
final_answer = ""
|
|
73
|
+
|
|
74
|
+
# Token counts are Optional in the SDK response
|
|
75
|
+
input_tokens = response.usage.prompt_tokens or 0
|
|
76
|
+
output_tokens = response.usage.completion_tokens or 0
|
|
66
77
|
|
|
67
78
|
# Calculate Cost
|
|
68
79
|
total_cost = calculate_cost(
|
|
@@ -101,10 +101,10 @@ class OllamaAdapter(AgentAdapter):
|
|
|
101
101
|
finish_reason = data.get("choices", [{}])[0].get("finish_reason", "stop")
|
|
102
102
|
|
|
103
103
|
# Build token usage
|
|
104
|
+
# total_tokens is a computed property on TokenUsage, not a field
|
|
104
105
|
token_usage = TokenUsage(
|
|
105
106
|
input_tokens=usage.get("prompt_tokens", 0),
|
|
106
107
|
output_tokens=usage.get("completion_tokens", 0),
|
|
107
|
-
total_tokens=usage.get("total_tokens", 0),
|
|
108
108
|
)
|
|
109
109
|
|
|
110
110
|
# Record LLM call span
|