evalview 0.8.1__tar.gz → 0.9.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {evalview-0.8.1/evalview.egg-info → evalview-0.9.1}/PKG-INFO +48 -9
- {evalview-0.8.1 → evalview-0.9.1}/README.md +41 -7
- {evalview-0.8.1 → evalview-0.9.1}/evalview/__init__.py +1 -1
- evalview-0.9.1/evalview/adapters/openai_assistants_adapter.py +473 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/adapters/pydantic_ai_adapter.py +102 -22
- {evalview-0.8.1 → evalview-0.9.1}/evalview/api.py +46 -37
- {evalview-0.8.1 → evalview-0.9.1}/evalview/chat.py +1 -1
- {evalview-0.8.1 → evalview-0.9.1}/evalview/chat_commands.py +1 -1
- {evalview-0.8.1 → evalview-0.9.1}/evalview/chat_prompt.py +37 -30
- {evalview-0.8.1 → evalview-0.9.1}/evalview/chat_session.py +43 -27
- {evalview-0.8.1 → evalview-0.9.1}/evalview/ci/comment.py +6 -3
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/_check_verdict.py +9 -1
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/badge_cmd.py +37 -53
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/check_cmd.py +186 -93
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/check_display.py +69 -5
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/drift_cmd.py +2 -2
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/listing_cmd.py +1 -1
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/monitor_cmd.py +57 -26
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/run/_adapters.py +30 -7
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/run/_reporter.py +1 -1
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/shared.py +42 -16
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/snapshot_cmd.py +27 -20
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/trends_cmd.py +16 -1
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/watch_cmd.py +13 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/adapter_factory.py +16 -1
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/config.py +16 -12
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/diff.py +16 -2
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/drift_tracker.py +44 -37
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/golden.py +106 -20
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/healing.py +32 -12
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/noise_tracker.py +7 -3
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/pricing.py +42 -15
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/recommendations.py +9 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/root_cause.py +10 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/runner.py +1 -1
- evalview-0.9.1/evalview/core/trends.py +91 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/types.py +33 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/verdict.py +10 -2
- {evalview-0.8.1 → evalview-0.9.1}/evalview/evaluators/tool_call_evaluator.py +36 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/mcp_server.py +226 -88
- {evalview-0.8.1 → evalview-0.9.1}/evalview/openclaw.py +12 -1
- {evalview-0.8.1 → evalview-0.9.1}/evalview/pytest_plugin.py +1 -1
- {evalview-0.8.1 → evalview-0.9.1}/evalview/skills/adapters/__init__.py +1 -1
- evalview-0.9.1/evalview/skills/adapters/openai_assistants_adapter.py +531 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/skills/adapters/registry.py +1 -1
- {evalview-0.8.1 → evalview-0.9.1}/evalview/tracking/__init__.py +2 -1
- {evalview-0.8.1 → evalview-0.9.1}/evalview/tracking/regression.py +61 -5
- {evalview-0.8.1 → evalview-0.9.1}/evalview/visualization/generators.py +10 -2
- {evalview-0.8.1 → evalview-0.9.1/evalview.egg-info}/PKG-INFO +48 -9
- {evalview-0.8.1 → evalview-0.9.1}/evalview.egg-info/SOURCES.txt +16 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview.egg-info/requires.txt +13 -1
- {evalview-0.8.1 → evalview-0.9.1}/pyproject.toml +20 -2
- evalview-0.9.1/tests/test_action_check_status.py +61 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_api.py +7 -4
- evalview-0.9.1/tests/test_chat_session.py +84 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_check_cmd.py +36 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_check_pipeline.py +190 -41
- evalview-0.9.1/tests/test_ci_provider_isolation.py +126 -0
- evalview-0.9.1/tests/test_documentation_links.py +87 -0
- evalview-0.9.1/tests/test_dogfood_ci.py +204 -0
- evalview-0.9.1/tests/test_dogfood_harness.py +122 -0
- evalview-0.9.1/tests/test_dogfood_issue_routing.py +112 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_drift_tracker.py +63 -1
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_evaluators.py +163 -3
- evalview-0.9.1/tests/test_forbidden_tool_gate.py +140 -0
- evalview-0.9.1/tests/test_incomplete_gate.py +210 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_mcp_server.py +339 -22
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_monitor.py +52 -0
- evalview-0.9.1/tests/test_openai_responses_adapter.py +555 -0
- evalview-0.9.1/tests/test_pricing.py +74 -0
- evalview-0.9.1/tests/test_pydantic_ai_adapter.py +126 -0
- evalview-0.9.1/tests/test_regression_tracker.py +279 -0
- evalview-0.9.1/tests/test_release_checks.py +167 -0
- evalview-0.9.1/tests/test_snapshot_reset.py +202 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_watch_cmd.py +12 -2
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_week3_progress_and_drift.py +2 -0
- evalview-0.8.1/evalview/adapters/openai_assistants_adapter.py +0 -476
- evalview-0.8.1/evalview/skills/adapters/openai_assistants_adapter.py +0 -637
- {evalview-0.8.1 → evalview-0.9.1}/LICENSE +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/NOTICE +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/__main__.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/adapters/__init__.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/adapters/aider_adapter.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/adapters/anthropic_adapter.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/adapters/base.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/adapters/cohere_adapter.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/adapters/crewai_adapter.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/adapters/crewai_native_adapter.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/adapters/goose_adapter.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/adapters/http_adapter.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/adapters/huggingface_adapter.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/adapters/langgraph_adapter.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/adapters/mcp_adapter.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/adapters/mistral_adapter.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/adapters/ollama_adapter.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/adapters/openclaw_adapter.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/adapters/opencode_adapter.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/adapters/registry.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/adapters/tapescope_adapter.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/adapters/vercel_ai_adapter.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/benchmarks/__init__.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/benchmarks/canary/README.md +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/benchmarks/canary/__init__.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/benchmarks/canary/suite.v1.held-out.yaml +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/benchmarks/canary/suite.v1.public.yaml +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/chat_demo.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/chat_runtime.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/chat_slash.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/ci/__init__.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/cli.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/cloud/__init__.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/cloud/auth.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/cloud/push.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/__init__.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/_check_helpers.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/_init_demo_agent.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/_init_tests.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/_init_wizard.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/add_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/assertion_wizard.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/autopr_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/baseline_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/benchmark_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/capture_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/chat_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/ci_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/cloud_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/demo_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/diff_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/expand_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/feedback_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/fleet_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/freshness_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/generate_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/golden_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/gym_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/hooks_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/import_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/init_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/judge_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/log_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/mcp_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/model_check_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/model_check_render.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/openclaw_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/progress_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/quarantine_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/replay_trace_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/run/__init__.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/run/_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/run/_executor.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/run/_helpers.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/run/_runner.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/simulate_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/since_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/skill_agent_runner.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/skill_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/slack_digest_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/telemetry_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/traces_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/validate_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/commands/visual_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/compare.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/__init__.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/behavioral_anomalies.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/benchmark_hardening.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/budget.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/canary_suite.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/cassette.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/celebrations.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/chaos.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/contract_diff.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/dashboard.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/discord_notifier.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/drift_classifier.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/drift_kind.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/eval_profiles.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/fleet.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/freshness.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/goal_drift.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/judge_cache.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/llm_configs.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/llm_provider.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/loader.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/mcp_contract.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/messages.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/model_check_scoring.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/model_provider_runner.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/model_runtime_detector.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/model_snapshots.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/observability.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/otel_semconv.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/parallel.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/project_state.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/quarantine.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/rationale.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/regression_synth.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/replay_pipeline.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/retrieval_lineage.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/retry.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/root_cause_hint.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/security.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/semantic_diff.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/simulation.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/slack_notifier.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/test_quality.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/text.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/tool_categories.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/tracing.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/turn_coherence.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/variant_clusterer.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/core/watcher.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/evaluators/__init__.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/evaluators/cost_evaluator.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/evaluators/evaluator.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/evaluators/hallucination_evaluator.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/evaluators/latency_evaluator.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/evaluators/output_evaluator.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/evaluators/pii_evaluator.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/evaluators/safety_evaluator.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/evaluators/sequence_evaluator.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/evaluators/statistical_evaluator.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/expander.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/exporters/__init__.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/exporters/html_exporter.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/importers/__init__.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/importers/log_importer.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/py.typed +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/recorder.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/reporters/__init__.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/reporters/_console_coverage.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/reporters/_console_statistical.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/reporters/_templates.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/reporters/console_reporter.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/reporters/html_reporter.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/reporters/json_reporter.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/reporters/trace_live_reporter.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/reporters/trace_reporter.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/skills/__init__.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/skills/adapters/base.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/skills/adapters/claude_code_adapter.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/skills/adapters/codex_adapter.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/skills/adapters/crewai_adapter.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/skills/adapters/custom_adapter.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/skills/adapters/langgraph_adapter.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/skills/adapters/openclaw_adapter.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/skills/agent_runner.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/skills/agent_types.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/skills/constants.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/skills/evaluators/__init__.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/skills/evaluators/_security_checks.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/skills/evaluators/_security_patterns.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/skills/evaluators/deterministic.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/skills/evaluators/orchestrator.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/skills/evaluators/rubric.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/skills/parser.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/skills/runner.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/skills/security_scanner.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/skills/test_generator.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/skills/test_helpers.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/skills/types.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/skills/ui_utils.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/skills/validator.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/storage/__init__.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/storage/database.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/telemetry/__init__.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/telemetry/client.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/telemetry/config.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/telemetry/decorators.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/telemetry/events.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/templates/patterns/code-generation.yaml +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/templates/patterns/cost-budget.yaml +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/templates/patterns/customer-support.yaml +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/templates/patterns/data-analysis.yaml +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/templates/patterns/latency-budget.yaml +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/templates/patterns/multi-turn-memory.yaml +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/templates/patterns/output-format.yaml +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/templates/patterns/rag-citation.yaml +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/templates/patterns/rag-grounding.yaml +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/templates/patterns/research-synthesis.yaml +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/templates/patterns/safety-refusal.yaml +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/templates/patterns/tool-error-handling.yaml +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/templates/patterns/tool-not-called.yaml +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/templates/patterns/tool-sequence.yaml +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/templates/patterns/wrong-tool-chosen.yaml +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/test_generation.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/test_generation_constants.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/test_generation_discovery.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/test_generation_seeds.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/test_generation_synthesis.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/test_generation_types.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/trace_cmd/__init__.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/trace_cmd/collector.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/trace_cmd/patcher.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/trace_cmd/runner.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/tracking/database.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/version_check.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/visualization/__init__.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/visualization/_mermaid.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview/visualization/_template.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview.egg-info/dependency_links.txt +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview.egg-info/entry_points.txt +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/evalview.egg-info/top_level.txt +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/setup.cfg +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_adapters.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_autopr.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_behavioral_anomalies.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_benchmark_hardening.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_canary_suite.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_cassette.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_celebrations.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_chaos.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_ci_generate_comment.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_cloud_auth_and_push.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_config_models.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_diff_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_diff_engine.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_dogfood_e2e.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_drift_classifier.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_e2e_snapshot_check.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_endpoint_detection.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_evaluator_accuracy.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_evaluator_orchestrator.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_fleet.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_forbidden_tools.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_freshness.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_generate_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_goal_drift.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_golden_store.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_healing.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_init_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_judge_cache.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_llm_provider.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_loader.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_loader_skill_suites.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_log_importer_csv.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_main_evaluator.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_mcp_contracts.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_messages.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_model_check_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_model_check_scoring.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_model_runtime_detector.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_model_snapshots.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_multi_turn_evaluation.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_new_features.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_noise_tracker.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_observability_regressions.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_otel_semconv.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_pii_evaluator.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_project_state.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_pytest_plugin.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_rationale.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_replay_pipeline.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_retrieval_lineage.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_root_cause.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_root_cause_hint.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_run_cmd_tags.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_run_guidance.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_runner.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_security.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_semantic_diff.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_simulation.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_snapshot_generated_workflow.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_snapshot_json_output.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_statistical_evaluator.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_telemetry_client.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_turn_coherence.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_types.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_validate_cmd.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_vercel_ai_adapter.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_verdict.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_version_check.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_visualization_generators.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_week1_integration.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_week2_quarantine.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_week2_review_fixes.py +0 -0
- {evalview-0.8.1 → evalview-0.9.1}/tests/test_week2_since_and_comment.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: evalview
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.9.1
|
|
4
4
|
Summary: Open-source testing and regression detection framework for AI agents. Golden baseline diffing, CI/CD integration, works with LangGraph, CrewAI, OpenAI, Anthropic Claude, HuggingFace, Ollama, and MCP.
|
|
5
5
|
Author-email: EvalView Team <hidai@evalview.com>
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -27,7 +27,7 @@ License-File: NOTICE
|
|
|
27
27
|
Requires-Dist: click>=8.1.0
|
|
28
28
|
Requires-Dist: pydantic>=2.5.0
|
|
29
29
|
Requires-Dist: pyyaml>=6.0
|
|
30
|
-
Requires-Dist: openai>=1.
|
|
30
|
+
Requires-Dist: openai>=1.101.0
|
|
31
31
|
Requires-Dist: anthropic>=0.39.0
|
|
32
32
|
Requires-Dist: rich>=13.7.0
|
|
33
33
|
Requires-Dist: prompt_toolkit>=3.0.0
|
|
@@ -36,6 +36,7 @@ Requires-Dist: python-dateutil>=2.8.2
|
|
|
36
36
|
Requires-Dist: python-dotenv>=1.0.0
|
|
37
37
|
Requires-Dist: jinja2>=3.0
|
|
38
38
|
Requires-Dist: jsonschema>=4.0.0
|
|
39
|
+
Requires-Dist: llm-prices>=1.4.0
|
|
39
40
|
Requires-Dist: tomli>=2.0; python_version < "3.11"
|
|
40
41
|
Provides-Extra: reports
|
|
41
42
|
Requires-Dist: plotly>=5.0; extra == "reports"
|
|
@@ -49,6 +50,8 @@ Provides-Extra: cohere
|
|
|
49
50
|
Requires-Dist: cohere>=5.0.0; extra == "cohere"
|
|
50
51
|
Provides-Extra: mistral
|
|
51
52
|
Requires-Dist: mistralai>=1.0.0; extra == "mistral"
|
|
53
|
+
Provides-Extra: pydantic-ai
|
|
54
|
+
Requires-Dist: pydantic-ai-slim>=0.0.30; python_version >= "3.10" and extra == "pydantic-ai"
|
|
52
55
|
Provides-Extra: all
|
|
53
56
|
Requires-Dist: plotly>=5.0; extra == "all"
|
|
54
57
|
Requires-Dist: watchdog>=3.0; extra == "all"
|
|
@@ -56,6 +59,7 @@ Requires-Dist: croniter>=2.0; extra == "all"
|
|
|
56
59
|
Requires-Dist: posthog>=3.0.0; extra == "all"
|
|
57
60
|
Requires-Dist: cohere>=5.0.0; extra == "all"
|
|
58
61
|
Requires-Dist: mistralai>=1.0.0; extra == "all"
|
|
62
|
+
Requires-Dist: pydantic-ai-slim>=0.0.30; python_version >= "3.10" and extra == "all"
|
|
59
63
|
Provides-Extra: dev
|
|
60
64
|
Requires-Dist: pytest>=7.4.0; extra == "dev"
|
|
61
65
|
Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
|
|
@@ -70,13 +74,14 @@ Requires-Dist: watchdog>=3.0; extra == "dev"
|
|
|
70
74
|
Requires-Dist: anthropic>=0.39.0; extra == "dev"
|
|
71
75
|
Requires-Dist: fastapi>=0.109.0; extra == "dev"
|
|
72
76
|
Requires-Dist: uvicorn>=0.27.0; extra == "dev"
|
|
77
|
+
Requires-Dist: pydantic-ai-slim>=0.0.30; python_version >= "3.10" and extra == "dev"
|
|
73
78
|
Dynamic: license-file
|
|
74
79
|
|
|
75
80
|
<!-- mcp-name: io.github.hidai25/evalview-mcp -->
|
|
76
81
|
<!-- keywords: AI agent testing, regression detection, golden baselines -->
|
|
77
82
|
|
|
78
83
|
<p align="center">
|
|
79
|
-
<img src="assets/logo.png" alt="EvalView" width="350">
|
|
84
|
+
<img src="https://raw.githubusercontent.com/hidai25/eval-view/v0.9.0/assets/logo.png" alt="EvalView" width="350">
|
|
80
85
|
<br>
|
|
81
86
|
<strong>Snapshot testing for AI agents.</strong><br>
|
|
82
87
|
Record what your agent does today. Get told when it silently changes.
|
|
@@ -85,7 +90,9 @@ Dynamic: license-file
|
|
|
85
90
|
<p align="center">
|
|
86
91
|
<a href="https://pypi.org/project/evalview/"><img src="https://img.shields.io/pypi/v/evalview.svg?label=release" alt="PyPI version"></a>
|
|
87
92
|
<a href="https://pypi.org/project/evalview/"><img src="https://img.shields.io/pypi/dm/evalview.svg?label=downloads" alt="PyPI downloads"></a>
|
|
88
|
-
<a href="https://github.com/hidai25/eval-view/actions/workflows/
|
|
93
|
+
<a href="https://github.com/hidai25/eval-view/actions/workflows/ci.yml"><img src="https://github.com/hidai25/eval-view/actions/workflows/ci.yml/badge.svg?branch=main" alt="Package CI"></a>
|
|
94
|
+
<a href="https://github.com/hidai25/eval-view/actions/workflows/dogfood.yml"><img src="https://github.com/hidai25/eval-view/actions/workflows/dogfood.yml/badge.svg?branch=main" alt="Core Dogfood"></a>
|
|
95
|
+
<a href="https://github.com/hidai25/eval-view/actions/workflows/dogfood-live.yml"><img src="https://github.com/hidai25/eval-view/actions/workflows/dogfood-live.yml/badge.svg?branch=main&event=workflow_dispatch" alt="Live Provider Checks (manual)"></a>
|
|
89
96
|
<a href="https://github.com/hidai25/eval-view/stargazers"><img src="https://img.shields.io/github/stars/hidai25/eval-view?style=social" alt="GitHub stars"></a>
|
|
90
97
|
<a href="https://opensource.org/licenses/Apache-2.0"><img src="https://img.shields.io/badge/License-Apache_2.0-blue.svg" alt="License"></a>
|
|
91
98
|
</p>
|
|
@@ -96,12 +103,17 @@ Your agent returns `200` and looks fine. But a model update, a provider change,
|
|
|
96
103
|
|
|
97
104
|
**EvalView snapshots your agent's behavior — the tools it calls, in what order, with what output — and tells you the moment that behavior changes.** Like Jest snapshots, but for tool-calling, multi-turn agents.
|
|
98
105
|
|
|
99
|
-
[](https://github.com/user-attachments/assets/96d8b5f7-3561-44a1-86a4-270fb0d1d8a6)
|
|
100
107
|
|
|
101
108
|
<sub>↑ 30-second live demo — no API key needed</sub>
|
|
102
109
|
|
|
103
110
|
## Quick Start
|
|
104
111
|
|
|
112
|
+
> **OpenAI adapter migration:** OpenAI shut down the Assistants API on **August 26, 2026**.
|
|
113
|
+
> EvalView 0.9.0 uses the Responses API. If you use `openai-assistants`, migrate your
|
|
114
|
+
> model, instructions and tools or configure a dashboard Prompt before running checks.
|
|
115
|
+
> [OpenAI migration guide](docs/OPENAI_MIGRATION.md)
|
|
116
|
+
|
|
105
117
|
```bash
|
|
106
118
|
pip install evalview
|
|
107
119
|
```
|
|
@@ -121,6 +133,9 @@ That's the whole loop. `check` returns one of:
|
|
|
121
133
|
|
|
122
134
|
It diffs the **whole trajectory** — tool names, parameters, and order — not just the final string. The deterministic tool + sequence diff runs offline, with no API key. Add an LLM judge only when you want output-quality scoring.
|
|
123
135
|
|
|
136
|
+
Executing your agent can still incur backend API charges: `--no-judge` skips the
|
|
137
|
+
judge, not those calls. Embedding-based semantic comparison is opt-in.
|
|
138
|
+
|
|
124
139
|
No agent yet? See it work in 30 seconds:
|
|
125
140
|
|
|
126
141
|
```bash
|
|
@@ -144,11 +159,29 @@ This makes EvalView a **merge-time regression gate**, which is a different job f
|
|
|
144
159
|
|
|
145
160
|
## EvalView tests itself in public, every day
|
|
146
161
|
|
|
147
|
-
|
|
162
|
+
Every day at 09:00 UTC, on pull requests, and on pushes to main,
|
|
163
|
+
[Core Dogfood](.github/workflows/dogfood.yml) exercises the non-live test suite,
|
|
164
|
+
type checks, local mock-agent `snapshot` / `check`, `evalview demo`, end-to-end
|
|
165
|
+
flows, and an `evalview monitor` smoke test. It uses no paid API credentials and
|
|
166
|
+
makes no paid inference calls. GitHub runner usage is separate.
|
|
167
|
+
|
|
168
|
+
[Live Provider Checks](.github/workflows/dogfood-live.yml) test the real evaluator and
|
|
169
|
+
chat assistant only when a maintainer explicitly opts into paid API use on main.
|
|
170
|
+
They have no automatic schedule. Their badge records the last manual run; a green
|
|
171
|
+
core badge does not establish live-provider health or rule out provider drift.
|
|
172
|
+
|
|
173
|
+
Package CI, core dogfood, and live checks have separate badges. Failed or incomplete
|
|
174
|
+
checks remain visible within their scope, with logs and reports preserved as
|
|
175
|
+
artifacts. Rolling issues use separate `dogfood-core` and `dogfood-live` labels.
|
|
176
|
+
A provider outage, exhausted quota, or missing credential means live health is
|
|
177
|
+
unavailable; it does not prove an agent regression.
|
|
148
178
|
|
|
149
|
-
|
|
179
|
+
The historical [incident #264](https://github.com/hidai25/eval-view/issues/264)
|
|
180
|
+
remains available for maintainer review of fresh evidence from both scopes. Neither
|
|
181
|
+
workflow automatically closes it. Trust warnings are evidence to investigate,
|
|
182
|
+
not proof of gaming or of a particular root cause.
|
|
150
183
|
|
|
151
|
-
[
|
|
184
|
+
[Core runs →](https://github.com/hidai25/eval-view/actions/workflows/dogfood.yml) · [Manual live runs →](https://github.com/hidai25/eval-view/actions/workflows/dogfood-live.yml) · [Run and triage guide →](docs/INTERNAL_DOGFOODING.md#daily-failure-triage)
|
|
152
185
|
|
|
153
186
|
## CI: block regressions in every PR
|
|
154
187
|
|
|
@@ -162,7 +195,7 @@ jobs:
|
|
|
162
195
|
permissions: { pull-requests: write }
|
|
163
196
|
steps:
|
|
164
197
|
- uses: actions/checkout@v4
|
|
165
|
-
- uses: hidai25/eval-view@v0.
|
|
198
|
+
- uses: hidai25/eval-view@v0.9.0
|
|
166
199
|
with:
|
|
167
200
|
openai-api-key: ${{ secrets.OPENAI_API_KEY }}
|
|
168
201
|
```
|
|
@@ -197,6 +230,12 @@ EvalView also does multi-turn testing, statistical/pass@k runs, record/replay ca
|
|
|
197
230
|
|
|
198
231
|
→ [Full feature reference](docs/CLI_REFERENCE.md) · [Getting Started](docs/GETTING_STARTED.md) · [FAQ](docs/FAQ.md)
|
|
199
232
|
|
|
233
|
+
→ [Documentation index](docs/README.md) · [OpenAI migration](docs/OPENAI_MIGRATION.md) · [Release process](docs/RELEASING.md)
|
|
234
|
+
|
|
235
|
+
### Why I built EvalView
|
|
236
|
+
|
|
237
|
+
An agent that looked successful kept pulling entire documents into its context and made one question cost $42.93. That experience led me to build EvalView. I wrote about it in [“I Was Running an AI Casino. Then I Started Writing Tests for My Agents”](https://medium.com/@hidaibarmor/i-was-running-an-ai-casino-then-i-started-writing-tests-for-my-agents-93cb3468ce1e). The December 2025 post is the origin story; use the current docs for setup and commands.
|
|
238
|
+
|
|
200
239
|
## Contributing
|
|
201
240
|
|
|
202
241
|
This is a young project built mostly by one developer. Issues, PRs, and "I tried it and X was confusing" feedback are all genuinely valuable.
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
<!-- keywords: AI agent testing, regression detection, golden baselines -->
|
|
3
3
|
|
|
4
4
|
<p align="center">
|
|
5
|
-
<img src="assets/logo.png" alt="EvalView" width="350">
|
|
5
|
+
<img src="https://raw.githubusercontent.com/hidai25/eval-view/v0.9.0/assets/logo.png" alt="EvalView" width="350">
|
|
6
6
|
<br>
|
|
7
7
|
<strong>Snapshot testing for AI agents.</strong><br>
|
|
8
8
|
Record what your agent does today. Get told when it silently changes.
|
|
@@ -11,7 +11,9 @@
|
|
|
11
11
|
<p align="center">
|
|
12
12
|
<a href="https://pypi.org/project/evalview/"><img src="https://img.shields.io/pypi/v/evalview.svg?label=release" alt="PyPI version"></a>
|
|
13
13
|
<a href="https://pypi.org/project/evalview/"><img src="https://img.shields.io/pypi/dm/evalview.svg?label=downloads" alt="PyPI downloads"></a>
|
|
14
|
-
<a href="https://github.com/hidai25/eval-view/actions/workflows/
|
|
14
|
+
<a href="https://github.com/hidai25/eval-view/actions/workflows/ci.yml"><img src="https://github.com/hidai25/eval-view/actions/workflows/ci.yml/badge.svg?branch=main" alt="Package CI"></a>
|
|
15
|
+
<a href="https://github.com/hidai25/eval-view/actions/workflows/dogfood.yml"><img src="https://github.com/hidai25/eval-view/actions/workflows/dogfood.yml/badge.svg?branch=main" alt="Core Dogfood"></a>
|
|
16
|
+
<a href="https://github.com/hidai25/eval-view/actions/workflows/dogfood-live.yml"><img src="https://github.com/hidai25/eval-view/actions/workflows/dogfood-live.yml/badge.svg?branch=main&event=workflow_dispatch" alt="Live Provider Checks (manual)"></a>
|
|
15
17
|
<a href="https://github.com/hidai25/eval-view/stargazers"><img src="https://img.shields.io/github/stars/hidai25/eval-view?style=social" alt="GitHub stars"></a>
|
|
16
18
|
<a href="https://opensource.org/licenses/Apache-2.0"><img src="https://img.shields.io/badge/License-Apache_2.0-blue.svg" alt="License"></a>
|
|
17
19
|
</p>
|
|
@@ -22,12 +24,17 @@ Your agent returns `200` and looks fine. But a model update, a provider change,
|
|
|
22
24
|
|
|
23
25
|
**EvalView snapshots your agent's behavior — the tools it calls, in what order, with what output — and tells you the moment that behavior changes.** Like Jest snapshots, but for tool-calling, multi-turn agents.
|
|
24
26
|
|
|
25
|
-
[](https://github.com/user-attachments/assets/96d8b5f7-3561-44a1-86a4-270fb0d1d8a6)
|
|
26
28
|
|
|
27
29
|
<sub>↑ 30-second live demo — no API key needed</sub>
|
|
28
30
|
|
|
29
31
|
## Quick Start
|
|
30
32
|
|
|
33
|
+
> **OpenAI adapter migration:** OpenAI shut down the Assistants API on **August 26, 2026**.
|
|
34
|
+
> EvalView 0.9.0 uses the Responses API. If you use `openai-assistants`, migrate your
|
|
35
|
+
> model, instructions and tools or configure a dashboard Prompt before running checks.
|
|
36
|
+
> [OpenAI migration guide](docs/OPENAI_MIGRATION.md)
|
|
37
|
+
|
|
31
38
|
```bash
|
|
32
39
|
pip install evalview
|
|
33
40
|
```
|
|
@@ -47,6 +54,9 @@ That's the whole loop. `check` returns one of:
|
|
|
47
54
|
|
|
48
55
|
It diffs the **whole trajectory** — tool names, parameters, and order — not just the final string. The deterministic tool + sequence diff runs offline, with no API key. Add an LLM judge only when you want output-quality scoring.
|
|
49
56
|
|
|
57
|
+
Executing your agent can still incur backend API charges: `--no-judge` skips the
|
|
58
|
+
judge, not those calls. Embedding-based semantic comparison is opt-in.
|
|
59
|
+
|
|
50
60
|
No agent yet? See it work in 30 seconds:
|
|
51
61
|
|
|
52
62
|
```bash
|
|
@@ -70,11 +80,29 @@ This makes EvalView a **merge-time regression gate**, which is a different job f
|
|
|
70
80
|
|
|
71
81
|
## EvalView tests itself in public, every day
|
|
72
82
|
|
|
73
|
-
|
|
83
|
+
Every day at 09:00 UTC, on pull requests, and on pushes to main,
|
|
84
|
+
[Core Dogfood](.github/workflows/dogfood.yml) exercises the non-live test suite,
|
|
85
|
+
type checks, local mock-agent `snapshot` / `check`, `evalview demo`, end-to-end
|
|
86
|
+
flows, and an `evalview monitor` smoke test. It uses no paid API credentials and
|
|
87
|
+
makes no paid inference calls. GitHub runner usage is separate.
|
|
88
|
+
|
|
89
|
+
[Live Provider Checks](.github/workflows/dogfood-live.yml) test the real evaluator and
|
|
90
|
+
chat assistant only when a maintainer explicitly opts into paid API use on main.
|
|
91
|
+
They have no automatic schedule. Their badge records the last manual run; a green
|
|
92
|
+
core badge does not establish live-provider health or rule out provider drift.
|
|
93
|
+
|
|
94
|
+
Package CI, core dogfood, and live checks have separate badges. Failed or incomplete
|
|
95
|
+
checks remain visible within their scope, with logs and reports preserved as
|
|
96
|
+
artifacts. Rolling issues use separate `dogfood-core` and `dogfood-live` labels.
|
|
97
|
+
A provider outage, exhausted quota, or missing credential means live health is
|
|
98
|
+
unavailable; it does not prove an agent regression.
|
|
74
99
|
|
|
75
|
-
|
|
100
|
+
The historical [incident #264](https://github.com/hidai25/eval-view/issues/264)
|
|
101
|
+
remains available for maintainer review of fresh evidence from both scopes. Neither
|
|
102
|
+
workflow automatically closes it. Trust warnings are evidence to investigate,
|
|
103
|
+
not proof of gaming or of a particular root cause.
|
|
76
104
|
|
|
77
|
-
[
|
|
105
|
+
[Core runs →](https://github.com/hidai25/eval-view/actions/workflows/dogfood.yml) · [Manual live runs →](https://github.com/hidai25/eval-view/actions/workflows/dogfood-live.yml) · [Run and triage guide →](docs/INTERNAL_DOGFOODING.md#daily-failure-triage)
|
|
78
106
|
|
|
79
107
|
## CI: block regressions in every PR
|
|
80
108
|
|
|
@@ -88,7 +116,7 @@ jobs:
|
|
|
88
116
|
permissions: { pull-requests: write }
|
|
89
117
|
steps:
|
|
90
118
|
- uses: actions/checkout@v4
|
|
91
|
-
- uses: hidai25/eval-view@v0.
|
|
119
|
+
- uses: hidai25/eval-view@v0.9.0
|
|
92
120
|
with:
|
|
93
121
|
openai-api-key: ${{ secrets.OPENAI_API_KEY }}
|
|
94
122
|
```
|
|
@@ -123,6 +151,12 @@ EvalView also does multi-turn testing, statistical/pass@k runs, record/replay ca
|
|
|
123
151
|
|
|
124
152
|
→ [Full feature reference](docs/CLI_REFERENCE.md) · [Getting Started](docs/GETTING_STARTED.md) · [FAQ](docs/FAQ.md)
|
|
125
153
|
|
|
154
|
+
→ [Documentation index](docs/README.md) · [OpenAI migration](docs/OPENAI_MIGRATION.md) · [Release process](docs/RELEASING.md)
|
|
155
|
+
|
|
156
|
+
### Why I built EvalView
|
|
157
|
+
|
|
158
|
+
An agent that looked successful kept pulling entire documents into its context and made one question cost $42.93. That experience led me to build EvalView. I wrote about it in [“I Was Running an AI Casino. Then I Started Writing Tests for My Agents”](https://medium.com/@hidaibarmor/i-was-running-an-ai-casino-then-i-started-writing-tests-for-my-agents-93cb3468ce1e). The December 2025 post is the origin story; use the current docs for setup and commands.
|
|
159
|
+
|
|
126
160
|
## Contributing
|
|
127
161
|
|
|
128
162
|
This is a young project built mostly by one developer. Issues, PRs, and "I tried it and X was confusing" feedback are all genuinely valuable.
|