evalview 0.7.0__tar.gz → 0.8.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {evalview-0.7.0/evalview.egg-info → evalview-0.8.0}/PKG-INFO +20 -4
- {evalview-0.7.0 → evalview-0.8.0}/README.md +15 -3
- {evalview-0.7.0 → evalview-0.8.0}/evalview/adapters/aider_adapter.py +0 -2
- {evalview-0.7.0 → evalview-0.8.0}/evalview/adapters/crewai_native_adapter.py +1 -3
- {evalview-0.7.0 → evalview-0.8.0}/evalview/adapters/pydantic_ai_adapter.py +1 -2
- evalview-0.8.0/evalview/chat.py +809 -0
- evalview-0.8.0/evalview/chat_commands.py +1010 -0
- evalview-0.8.0/evalview/chat_demo.py +562 -0
- evalview-0.8.0/evalview/chat_prompt.py +479 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/chat_runtime.py +41 -13
- evalview-0.8.0/evalview/chat_session.py +149 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/cli.py +78 -63
- {evalview-0.7.0 → evalview-0.8.0}/evalview/cloud/push.py +38 -2
- evalview-0.8.0/evalview/commands/_check_helpers.py +199 -0
- evalview-0.8.0/evalview/commands/_check_verdict.py +373 -0
- evalview-0.8.0/evalview/commands/_init_demo_agent.py +178 -0
- evalview-0.8.0/evalview/commands/_init_tests.py +280 -0
- evalview-0.8.0/evalview/commands/_init_wizard.py +192 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/assertion_wizard.py +2 -7
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/autopr_cmd.py +0 -1
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/badge_cmd.py +2 -3
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/baseline_cmd.py +1 -1
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/benchmark_cmd.py +1 -1
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/check_cmd.py +26 -534
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/check_display.py +1 -2
- evalview-0.8.0/evalview/commands/diff_cmd.py +283 -0
- evalview-0.8.0/evalview/commands/fleet_cmd.py +223 -0
- evalview-0.8.0/evalview/commands/freshness_cmd.py +418 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/generate_cmd.py +1 -1
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/golden_cmd.py +1 -1
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/gym_cmd.py +1 -1
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/hooks_cmd.py +2 -2
- evalview-0.8.0/evalview/commands/init_cmd.py +564 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/judge_cmd.py +1 -1
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/listing_cmd.py +48 -10
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/log_cmd.py +2 -2
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/model_check_cmd.py +0 -1
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/monitor_cmd.py +104 -7
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/openclaw_cmd.py +1 -2
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/quarantine_cmd.py +1 -2
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/replay_trace_cmd.py +1 -2
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/run/_cmd.py +12 -335
- evalview-0.8.0/evalview/commands/run/_helpers.py +360 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/simulate_cmd.py +112 -3
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/since_cmd.py +2 -2
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/slack_digest_cmd.py +1 -2
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/telemetry_cmd.py +1 -1
- evalview-0.8.0/evalview/commands/validate_cmd.py +209 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/visual_cmd.py +13 -56
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/behavioral_anomalies.py +1 -1
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/benchmark_hardening.py +27 -2
- evalview-0.8.0/evalview/core/cassette.py +208 -0
- evalview-0.8.0/evalview/core/chaos.py +294 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/config.py +7 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/discord_notifier.py +19 -3
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/eval_profiles.py +1 -4
- evalview-0.8.0/evalview/core/fleet.py +442 -0
- evalview-0.8.0/evalview/core/freshness.py +500 -0
- evalview-0.8.0/evalview/core/goal_drift.py +259 -0
- evalview-0.8.0/evalview/core/loader.py +126 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/model_provider_runner.py +2 -2
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/noise_tracker.py +24 -2
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/observability.py +12 -2
- evalview-0.8.0/evalview/core/otel_semconv.py +307 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/quarantine.py +1 -2
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/rationale.py +13 -1
- evalview-0.8.0/evalview/core/retrieval_lineage.py +318 -0
- evalview-0.8.0/evalview/core/root_cause_hint.py +485 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/simulation.py +209 -8
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/slack_notifier.py +22 -4
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/turn_coherence.py +1 -1
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/types.py +38 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/evaluators/hallucination_evaluator.py +0 -1
- {evalview-0.7.0 → evalview-0.8.0}/evalview/evaluators/output_evaluator.py +0 -1
- {evalview-0.7.0 → evalview-0.8.0}/evalview/importers/log_importer.py +121 -6
- {evalview-0.7.0 → evalview-0.8.0}/evalview/pytest_plugin.py +2 -2
- evalview-0.8.0/evalview/reporters/_console_coverage.py +182 -0
- evalview-0.8.0/evalview/reporters/_console_statistical.py +313 -0
- evalview-0.7.0/evalview/reporters/html_reporter.py → evalview-0.8.0/evalview/reporters/_templates.py +7 -633
- {evalview-0.7.0 → evalview-0.8.0}/evalview/reporters/console_reporter.py +3 -464
- evalview-0.8.0/evalview/reporters/html_reporter.py +637 -0
- evalview-0.8.0/evalview/skills/evaluators/_security_checks.py +379 -0
- evalview-0.8.0/evalview/skills/evaluators/_security_patterns.py +89 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/skills/evaluators/deterministic.py +8 -432
- {evalview-0.7.0 → evalview-0.8.0}/evalview/test_generation.py +35 -866
- evalview-0.8.0/evalview/test_generation_constants.py +188 -0
- evalview-0.8.0/evalview/test_generation_discovery.py +158 -0
- evalview-0.8.0/evalview/test_generation_seeds.py +260 -0
- evalview-0.8.0/evalview/test_generation_synthesis.py +391 -0
- evalview-0.8.0/evalview/test_generation_types.py +53 -0
- evalview-0.8.0/evalview/visualization/_mermaid.py +96 -0
- evalview-0.7.0/evalview/visualization/generators.py → evalview-0.8.0/evalview/visualization/_template.py +3 -972
- evalview-0.8.0/evalview/visualization/generators.py +893 -0
- {evalview-0.7.0 → evalview-0.8.0/evalview.egg-info}/PKG-INFO +20 -4
- {evalview-0.7.0 → evalview-0.8.0}/evalview.egg-info/SOURCES.txt +45 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview.egg-info/requires.txt +7 -0
- {evalview-0.7.0 → evalview-0.8.0}/pyproject.toml +9 -1
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_api.py +2 -8
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_autopr.py +1 -4
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_behavioral_anomalies.py +1 -6
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_benchmark_hardening.py +40 -4
- evalview-0.8.0/tests/test_cassette.py +364 -0
- evalview-0.8.0/tests/test_chaos.py +187 -0
- evalview-0.8.0/tests/test_diff_cmd.py +237 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_dogfood_e2e.py +5 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_drift_classifier.py +0 -1
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_evaluator_accuracy.py +2 -3
- evalview-0.8.0/tests/test_fleet.py +399 -0
- evalview-0.8.0/tests/test_freshness.py +541 -0
- evalview-0.8.0/tests/test_goal_drift.py +222 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_healing.py +1 -4
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_init_cmd.py +18 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_loader.py +79 -0
- evalview-0.8.0/tests/test_log_importer_csv.py +132 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_mcp_server.py +3 -4
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_monitor.py +126 -1
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_multi_turn_evaluation.py +0 -2
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_noise_tracker.py +0 -1
- evalview-0.8.0/tests/test_otel_semconv.py +119 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_rationale.py +22 -1
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_replay_pipeline.py +0 -1
- evalview-0.8.0/tests/test_retrieval_lineage.py +259 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_root_cause.py +0 -1
- evalview-0.8.0/tests/test_root_cause_hint.py +403 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_simulation.py +96 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_snapshot_generated_workflow.py +14 -1
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_turn_coherence.py +1 -6
- evalview-0.8.0/tests/test_validate_cmd.py +178 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_week1_integration.py +5 -5
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_week2_quarantine.py +1 -1
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_week2_review_fixes.py +0 -2
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_week2_since_and_comment.py +2 -2
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_week3_progress_and_drift.py +0 -1
- evalview-0.7.0/evalview/chat.py +0 -2840
- evalview-0.7.0/evalview/commands/init_cmd.py +0 -1549
- evalview-0.7.0/evalview/core/loader.py +0 -64
- {evalview-0.7.0 → evalview-0.8.0}/LICENSE +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/NOTICE +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/__init__.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/__main__.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/adapters/__init__.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/adapters/anthropic_adapter.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/adapters/base.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/adapters/cohere_adapter.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/adapters/crewai_adapter.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/adapters/goose_adapter.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/adapters/http_adapter.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/adapters/huggingface_adapter.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/adapters/langgraph_adapter.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/adapters/mcp_adapter.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/adapters/mistral_adapter.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/adapters/ollama_adapter.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/adapters/openai_assistants_adapter.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/adapters/openclaw_adapter.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/adapters/opencode_adapter.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/adapters/registry.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/adapters/tapescope_adapter.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/api.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/benchmarks/__init__.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/benchmarks/canary/README.md +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/benchmarks/canary/__init__.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/benchmarks/canary/suite.v1.held-out.yaml +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/benchmarks/canary/suite.v1.public.yaml +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/chat_slash.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/ci/__init__.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/ci/comment.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/cloud/__init__.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/cloud/auth.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/cloud/client.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/__init__.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/add_cmd.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/capture_cmd.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/chat_cmd.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/ci_cmd.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/cloud_cmd.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/demo_cmd.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/drift_cmd.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/expand_cmd.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/feedback_cmd.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/import_cmd.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/mcp_cmd.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/model_check_render.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/progress_cmd.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/run/__init__.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/run/_adapters.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/run/_executor.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/run/_reporter.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/run/_runner.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/shared.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/skill_agent_runner.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/skill_cmd.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/snapshot_cmd.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/traces_cmd.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/trends_cmd.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/commands/watch_cmd.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/compare.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/__init__.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/adapter_factory.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/budget.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/canary_suite.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/celebrations.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/contract_diff.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/dashboard.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/diff.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/drift_classifier.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/drift_kind.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/drift_tracker.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/golden.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/healing.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/judge_cache.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/llm_configs.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/llm_provider.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/mcp_contract.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/messages.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/model_check_scoring.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/model_runtime_detector.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/model_snapshots.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/parallel.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/pricing.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/project_state.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/recommendations.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/regression_synth.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/replay_pipeline.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/retry.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/root_cause.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/runner.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/security.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/semantic_diff.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/test_quality.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/tool_categories.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/tracing.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/variant_clusterer.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/verdict.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/core/watcher.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/evaluators/__init__.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/evaluators/cost_evaluator.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/evaluators/evaluator.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/evaluators/latency_evaluator.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/evaluators/pii_evaluator.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/evaluators/safety_evaluator.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/evaluators/sequence_evaluator.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/evaluators/statistical_evaluator.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/evaluators/tool_call_evaluator.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/expander.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/exporters/__init__.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/exporters/html_exporter.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/importers/__init__.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/mcp_server.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/openclaw.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/recorder.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/reporters/__init__.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/reporters/json_reporter.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/reporters/trace_live_reporter.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/reporters/trace_reporter.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/skills/__init__.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/skills/adapters/__init__.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/skills/adapters/base.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/skills/adapters/claude_code_adapter.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/skills/adapters/codex_adapter.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/skills/adapters/crewai_adapter.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/skills/adapters/custom_adapter.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/skills/adapters/langgraph_adapter.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/skills/adapters/openai_assistants_adapter.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/skills/adapters/openclaw_adapter.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/skills/adapters/registry.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/skills/agent_runner.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/skills/agent_types.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/skills/constants.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/skills/evaluators/__init__.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/skills/evaluators/orchestrator.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/skills/evaluators/rubric.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/skills/parser.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/skills/runner.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/skills/security_scanner.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/skills/test_generator.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/skills/test_helpers.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/skills/types.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/skills/ui_utils.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/skills/validator.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/storage/__init__.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/storage/database.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/telemetry/__init__.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/telemetry/client.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/telemetry/config.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/telemetry/decorators.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/telemetry/events.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/templates/patterns/code-generation.yaml +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/templates/patterns/cost-budget.yaml +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/templates/patterns/customer-support.yaml +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/templates/patterns/data-analysis.yaml +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/templates/patterns/latency-budget.yaml +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/templates/patterns/multi-turn-memory.yaml +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/templates/patterns/output-format.yaml +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/templates/patterns/rag-citation.yaml +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/templates/patterns/rag-grounding.yaml +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/templates/patterns/research-synthesis.yaml +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/templates/patterns/safety-refusal.yaml +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/templates/patterns/tool-error-handling.yaml +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/templates/patterns/tool-not-called.yaml +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/templates/patterns/tool-sequence.yaml +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/templates/patterns/wrong-tool-chosen.yaml +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/trace_cmd/__init__.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/trace_cmd/collector.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/trace_cmd/patcher.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/trace_cmd/runner.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/tracking/__init__.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/tracking/database.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/tracking/regression.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/version_check.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview/visualization/__init__.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview.egg-info/dependency_links.txt +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview.egg-info/entry_points.txt +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/evalview.egg-info/top_level.txt +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/setup.cfg +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_adapters.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_canary_suite.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_celebrations.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_check_cmd.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_check_pipeline.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_ci_generate_comment.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_config_models.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_diff_engine.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_drift_tracker.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_e2e_snapshot_check.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_endpoint_detection.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_evaluator_orchestrator.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_evaluators.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_forbidden_tools.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_generate_cmd.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_golden_store.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_judge_cache.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_llm_provider.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_main_evaluator.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_mcp_contracts.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_messages.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_model_check_cmd.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_model_check_scoring.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_model_runtime_detector.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_model_snapshots.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_new_features.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_observability_regressions.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_pii_evaluator.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_project_state.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_pytest_plugin.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_run_cmd_tags.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_run_guidance.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_runner.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_security.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_semantic_diff.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_snapshot_json_output.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_statistical_evaluator.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_telemetry_client.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_types.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_verdict.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_version_check.py +0 -0
- {evalview-0.7.0 → evalview-0.8.0}/tests/test_visualization_generators.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: evalview
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.8.0
|
|
4
4
|
Summary: Open-source testing and regression detection framework for AI agents. Golden baseline diffing, CI/CD integration, works with LangGraph, CrewAI, OpenAI, Anthropic Claude, HuggingFace, Ollama, and MCP.
|
|
5
5
|
Author-email: EvalView Team <hidai@evalview.com>
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -36,10 +36,13 @@ Requires-Dist: python-dateutil>=2.8.2
|
|
|
36
36
|
Requires-Dist: python-dotenv>=1.0.0
|
|
37
37
|
Requires-Dist: jinja2>=3.0
|
|
38
38
|
Requires-Dist: jsonschema>=4.0.0
|
|
39
|
+
Requires-Dist: tomli>=2.0; python_version < "3.11"
|
|
39
40
|
Provides-Extra: reports
|
|
40
41
|
Requires-Dist: plotly>=5.0; extra == "reports"
|
|
41
42
|
Provides-Extra: watch
|
|
42
43
|
Requires-Dist: watchdog>=3.0; extra == "watch"
|
|
44
|
+
Provides-Extra: schedule
|
|
45
|
+
Requires-Dist: croniter>=2.0; extra == "schedule"
|
|
43
46
|
Provides-Extra: telemetry
|
|
44
47
|
Requires-Dist: posthog>=3.0.0; extra == "telemetry"
|
|
45
48
|
Provides-Extra: cohere
|
|
@@ -49,6 +52,7 @@ Requires-Dist: mistralai>=1.0.0; extra == "mistral"
|
|
|
49
52
|
Provides-Extra: all
|
|
50
53
|
Requires-Dist: plotly>=5.0; extra == "all"
|
|
51
54
|
Requires-Dist: watchdog>=3.0; extra == "all"
|
|
55
|
+
Requires-Dist: croniter>=2.0; extra == "all"
|
|
52
56
|
Requires-Dist: posthog>=3.0.0; extra == "all"
|
|
53
57
|
Requires-Dist: cohere>=5.0.0; extra == "all"
|
|
54
58
|
Requires-Dist: mistralai>=1.0.0; extra == "all"
|
|
@@ -105,6 +109,7 @@ Most eval tools stop at detect and compare. EvalView helps you classify changes,
|
|
|
105
109
|
- Catch silent regressions that normal tests miss
|
|
106
110
|
- Separate provider/model drift from real system regressions
|
|
107
111
|
- Auto-heal flaky failures with retries, review gates, and audit logs
|
|
112
|
+
- Replay deterministically — cassettes capture real tool calls once so CI never re-hits live services
|
|
108
113
|
|
|
109
114
|
Built for **frontier-lab rigor, startup-team practicality**:
|
|
110
115
|
- targeted behavior runs instead of giant always-on eval suites
|
|
@@ -188,7 +193,7 @@ evalview model-check --model claude-opus-4-5-20251101 # next week, detects any
|
|
|
188
193
|
<summary><strong>Other install methods</strong></summary>
|
|
189
194
|
|
|
190
195
|
```bash
|
|
191
|
-
curl -fsSL https://raw.githubusercontent.com/hidai25/eval-view/main/install.sh | bash
|
|
196
|
+
curl -fsSL https://raw.githubusercontent.com/hidai25/eval-view/main/scripts/install.sh | bash
|
|
192
197
|
```
|
|
193
198
|
|
|
194
199
|
</details>
|
|
@@ -339,6 +344,7 @@ Use LangSmith for observability. Use Braintrust for scoring. **Use EvalView for
|
|
|
339
344
|
| Golden baseline regression | — | Manual | — | **Automatic** |
|
|
340
345
|
| Silent model change detection | — | — | — | **Yes** |
|
|
341
346
|
| Auto-heal (retry + variant proposal) | — | — | — | **Yes** |
|
|
347
|
+
| Hermetic record/replay | — | — | — | **Yes** |
|
|
342
348
|
| PR comments with alerts | — | — | — | **Cost, latency, model change** |
|
|
343
349
|
| Works without API keys | No | No | Partial | **Yes** |
|
|
344
350
|
| Production monitoring | Tracing | — | — | **Check loop + Slack** |
|
|
@@ -774,6 +780,15 @@ Two features that close the gaps the April 2026 agent-eval reports kept calling
|
|
|
774
780
|
evalview simulate tests/ --variants 5 --seed 42
|
|
775
781
|
```
|
|
776
782
|
|
|
783
|
+
**Record/replay cassettes** — declarative mocks are tedious for agents with dozens of tools. Record once against the real backend, replay forever — no network, no LLM cost beyond the agent's own model calls, no flaky third-party APIs in CI. *"The agent saw X, called Y, got Z"* becomes deterministic without re-hitting live services.
|
|
784
|
+
|
|
785
|
+
```bash
|
|
786
|
+
evalview simulate tests/refund.yaml --record # capture real tool results once
|
|
787
|
+
evalview simulate tests/refund.yaml --replay # hermetic from now on
|
|
788
|
+
```
|
|
789
|
+
|
|
790
|
+
Cassettes live at `.evalview/cassettes/<test>.json`, use per-tool sequential matching (robust to inter-tool ordering drift), and are versioned for forward compatibility. Declarative `mocks:` still take precedence so you can override a single recording without re-recording the whole run.
|
|
791
|
+
|
|
777
792
|
**Decision rationale** — every tool_choice / branch gets recorded with the chosen option, alternatives considered, and any model-reported reasoning (Anthropic `thinking` blocks auto-captured). Grouped across runs by `input_hash` so cloud analytics surfaces decision drift before your users notice. Local HTML replay shows it inline.
|
|
778
793
|
|
|
779
794
|
Supported adapters: Anthropic, OpenAI Assistants, LangGraph, CrewAI native.
|
|
@@ -792,6 +807,7 @@ Supported adapters: Anthropic, OpenAI Assistants, LangGraph, CrewAI native.
|
|
|
792
807
|
| **Recommendation engine** | Suggests the next command from verdict, drift class, and history | [Above](#daily-workflow) |
|
|
793
808
|
| **Model drift detection** | `model-check` — zero-judge canary suite that catches silent model updates | [Docs](#model-drift-detection) |
|
|
794
809
|
| **Simulation harness** | `evalview simulate` — hermetic what-if runs against declared mocks, with `--variants N` fan-out | [Docs](docs/SIMULATE.md) |
|
|
810
|
+
| **Record/replay cassettes** | Capture real tool calls once, replay deterministically forever — no live services in CI | [Docs](docs/SIMULATE.md#record--replay-cassettes) |
|
|
795
811
|
| **Decision rationale** | Structured `tool_choice` / `branch` logging with cross-run grouping for decision-drift detection | [Docs](docs/RATIONALE.md) |
|
|
796
812
|
| **Assertion wizard** | Analyze captured traffic, suggest smart assertions automatically | [Above](#assertion-wizard--tests-from-real-traffic) |
|
|
797
813
|
| **Auto-variant discovery** | Run N times, cluster paths, save valid variants | [Above](#auto-variant-discovery--solve-non-determinism) |
|
|
@@ -1011,7 +1027,7 @@ Then just ask Claude: "did my refactor break anything?" and it runs `run_check`
|
|
|
1011
1027
|
|
|
1012
1028
|
**Works with your coding agent out of the box.** Ask Cursor, Claude Code, or Copilot to add regression tests, build a new adapter, or debug a failing check — EvalView ships the architecture maps and task recipes they need to get it right on the first try.
|
|
1013
1029
|
|
|
1014
|
-
- [
|
|
1030
|
+
- [AGENTS.md](AGENTS.md) — architecture map, contracts, invariants, verification commands
|
|
1015
1031
|
- [Agent Recipes](docs/agent-recipes/README.md) — task-specific playbooks for common extensions
|
|
1016
1032
|
- [Add an Adapter](docs/agent-recipes/add-adapter.md)
|
|
1017
1033
|
- [Add an Evaluator](docs/agent-recipes/add-evaluator.md)
|
|
@@ -1026,7 +1042,7 @@ Then just ask Claude: "did my refactor break anything?" and it runs `run_check`
|
|
|
1026
1042
|
| [Getting Started](docs/GETTING_STARTED.md) | [Golden Traces](docs/GOLDEN_TRACES.md) | [CI/CD](docs/CI_CD.md) |
|
|
1027
1043
|
| [CLI Reference](docs/CLI_REFERENCE.md) | [Evaluation Metrics](docs/EVALUATION_METRICS.md) | [MCP Contracts](docs/MCP_CONTRACTS.md) |
|
|
1028
1044
|
| [Simulation](docs/SIMULATE.md) | [Decision Rationale](docs/RATIONALE.md) | |
|
|
1029
|
-
| [Agent Instructions](
|
|
1045
|
+
| [Agent Instructions](AGENTS.md) | [Agent Recipes](docs/agent-recipes/README.md) | [Ollama Recipe](docs/agent-recipes/integrate-ollama.md) |
|
|
1030
1046
|
| [FAQ](docs/FAQ.md) | [Test Generation](docs/TEST_GENERATION.md) | [Skills Testing](docs/SKILLS_TESTING.md) |
|
|
1031
1047
|
| [YAML Schema](docs/YAML_SCHEMA.md) | [Statistical Mode](docs/STATISTICAL_MODE.md) | [Chat Mode](docs/CHAT_MODE.md) |
|
|
1032
1048
|
| [Framework Support](docs/FRAMEWORK_SUPPORT.md) | [Behavior Coverage](docs/BEHAVIOR_COVERAGE.md) | [Debugging](docs/DEBUGGING.md) |
|
|
@@ -35,6 +35,7 @@ Most eval tools stop at detect and compare. EvalView helps you classify changes,
|
|
|
35
35
|
- Catch silent regressions that normal tests miss
|
|
36
36
|
- Separate provider/model drift from real system regressions
|
|
37
37
|
- Auto-heal flaky failures with retries, review gates, and audit logs
|
|
38
|
+
- Replay deterministically — cassettes capture real tool calls once so CI never re-hits live services
|
|
38
39
|
|
|
39
40
|
Built for **frontier-lab rigor, startup-team practicality**:
|
|
40
41
|
- targeted behavior runs instead of giant always-on eval suites
|
|
@@ -118,7 +119,7 @@ evalview model-check --model claude-opus-4-5-20251101 # next week, detects any
|
|
|
118
119
|
<summary><strong>Other install methods</strong></summary>
|
|
119
120
|
|
|
120
121
|
```bash
|
|
121
|
-
curl -fsSL https://raw.githubusercontent.com/hidai25/eval-view/main/install.sh | bash
|
|
122
|
+
curl -fsSL https://raw.githubusercontent.com/hidai25/eval-view/main/scripts/install.sh | bash
|
|
122
123
|
```
|
|
123
124
|
|
|
124
125
|
</details>
|
|
@@ -269,6 +270,7 @@ Use LangSmith for observability. Use Braintrust for scoring. **Use EvalView for
|
|
|
269
270
|
| Golden baseline regression | — | Manual | — | **Automatic** |
|
|
270
271
|
| Silent model change detection | — | — | — | **Yes** |
|
|
271
272
|
| Auto-heal (retry + variant proposal) | — | — | — | **Yes** |
|
|
273
|
+
| Hermetic record/replay | — | — | — | **Yes** |
|
|
272
274
|
| PR comments with alerts | — | — | — | **Cost, latency, model change** |
|
|
273
275
|
| Works without API keys | No | No | Partial | **Yes** |
|
|
274
276
|
| Production monitoring | Tracing | — | — | **Check loop + Slack** |
|
|
@@ -704,6 +706,15 @@ Two features that close the gaps the April 2026 agent-eval reports kept calling
|
|
|
704
706
|
evalview simulate tests/ --variants 5 --seed 42
|
|
705
707
|
```
|
|
706
708
|
|
|
709
|
+
**Record/replay cassettes** — declarative mocks are tedious for agents with dozens of tools. Record once against the real backend, replay forever — no network, no LLM cost beyond the agent's own model calls, no flaky third-party APIs in CI. *"The agent saw X, called Y, got Z"* becomes deterministic without re-hitting live services.
|
|
710
|
+
|
|
711
|
+
```bash
|
|
712
|
+
evalview simulate tests/refund.yaml --record # capture real tool results once
|
|
713
|
+
evalview simulate tests/refund.yaml --replay # hermetic from now on
|
|
714
|
+
```
|
|
715
|
+
|
|
716
|
+
Cassettes live at `.evalview/cassettes/<test>.json`, use per-tool sequential matching (robust to inter-tool ordering drift), and are versioned for forward compatibility. Declarative `mocks:` still take precedence so you can override a single recording without re-recording the whole run.
|
|
717
|
+
|
|
707
718
|
**Decision rationale** — every tool_choice / branch gets recorded with the chosen option, alternatives considered, and any model-reported reasoning (Anthropic `thinking` blocks auto-captured). Grouped across runs by `input_hash` so cloud analytics surfaces decision drift before your users notice. Local HTML replay shows it inline.
|
|
708
719
|
|
|
709
720
|
Supported adapters: Anthropic, OpenAI Assistants, LangGraph, CrewAI native.
|
|
@@ -722,6 +733,7 @@ Supported adapters: Anthropic, OpenAI Assistants, LangGraph, CrewAI native.
|
|
|
722
733
|
| **Recommendation engine** | Suggests the next command from verdict, drift class, and history | [Above](#daily-workflow) |
|
|
723
734
|
| **Model drift detection** | `model-check` — zero-judge canary suite that catches silent model updates | [Docs](#model-drift-detection) |
|
|
724
735
|
| **Simulation harness** | `evalview simulate` — hermetic what-if runs against declared mocks, with `--variants N` fan-out | [Docs](docs/SIMULATE.md) |
|
|
736
|
+
| **Record/replay cassettes** | Capture real tool calls once, replay deterministically forever — no live services in CI | [Docs](docs/SIMULATE.md#record--replay-cassettes) |
|
|
725
737
|
| **Decision rationale** | Structured `tool_choice` / `branch` logging with cross-run grouping for decision-drift detection | [Docs](docs/RATIONALE.md) |
|
|
726
738
|
| **Assertion wizard** | Analyze captured traffic, suggest smart assertions automatically | [Above](#assertion-wizard--tests-from-real-traffic) |
|
|
727
739
|
| **Auto-variant discovery** | Run N times, cluster paths, save valid variants | [Above](#auto-variant-discovery--solve-non-determinism) |
|
|
@@ -941,7 +953,7 @@ Then just ask Claude: "did my refactor break anything?" and it runs `run_check`
|
|
|
941
953
|
|
|
942
954
|
**Works with your coding agent out of the box.** Ask Cursor, Claude Code, or Copilot to add regression tests, build a new adapter, or debug a failing check — EvalView ships the architecture maps and task recipes they need to get it right on the first try.
|
|
943
955
|
|
|
944
|
-
- [
|
|
956
|
+
- [AGENTS.md](AGENTS.md) — architecture map, contracts, invariants, verification commands
|
|
945
957
|
- [Agent Recipes](docs/agent-recipes/README.md) — task-specific playbooks for common extensions
|
|
946
958
|
- [Add an Adapter](docs/agent-recipes/add-adapter.md)
|
|
947
959
|
- [Add an Evaluator](docs/agent-recipes/add-evaluator.md)
|
|
@@ -956,7 +968,7 @@ Then just ask Claude: "did my refactor break anything?" and it runs `run_check`
|
|
|
956
968
|
| [Getting Started](docs/GETTING_STARTED.md) | [Golden Traces](docs/GOLDEN_TRACES.md) | [CI/CD](docs/CI_CD.md) |
|
|
957
969
|
| [CLI Reference](docs/CLI_REFERENCE.md) | [Evaluation Metrics](docs/EVALUATION_METRICS.md) | [MCP Contracts](docs/MCP_CONTRACTS.md) |
|
|
958
970
|
| [Simulation](docs/SIMULATE.md) | [Decision Rationale](docs/RATIONALE.md) | |
|
|
959
|
-
| [Agent Instructions](
|
|
971
|
+
| [Agent Instructions](AGENTS.md) | [Agent Recipes](docs/agent-recipes/README.md) | [Ollama Recipe](docs/agent-recipes/integrate-ollama.md) |
|
|
960
972
|
| [FAQ](docs/FAQ.md) | [Test Generation](docs/TEST_GENERATION.md) | [Skills Testing](docs/SKILLS_TESTING.md) |
|
|
961
973
|
| [YAML Schema](docs/YAML_SCHEMA.md) | [Statistical Mode](docs/STATISTICAL_MODE.md) | [Chat Mode](docs/CHAT_MODE.md) |
|
|
962
974
|
| [Framework Support](docs/FRAMEWORK_SUPPORT.md) | [Behavior Coverage](docs/BEHAVIOR_COVERAGE.md) | [Debugging](docs/DEBUGGING.md) |
|
|
@@ -249,8 +249,6 @@ class AiderAdapter(AgentAdapter):
|
|
|
249
249
|
logger.warning("Failed to restore %s: %s", path, exc)
|
|
250
250
|
# Remove files that didn't exist before the run
|
|
251
251
|
if snapshots:
|
|
252
|
-
roots = {next(iter(snapshots)).parent}
|
|
253
|
-
# Walk from any snapshot's parent up to find the common root
|
|
254
252
|
common_root = Path(os.path.commonpath([str(p) for p in snapshots]))
|
|
255
253
|
if common_root.is_dir():
|
|
256
254
|
for path in common_root.rglob("*"):
|
|
@@ -26,7 +26,7 @@ import logging
|
|
|
26
26
|
import time
|
|
27
27
|
import uuid
|
|
28
28
|
from datetime import datetime, timezone
|
|
29
|
-
from typing import Any, Dict, List, Optional
|
|
29
|
+
from typing import Any, Dict, List, Optional
|
|
30
30
|
|
|
31
31
|
from evalview.adapters.base import AgentAdapter
|
|
32
32
|
from evalview.core.types import (
|
|
@@ -232,8 +232,6 @@ class CrewAINativeAdapter(AgentAdapter):
|
|
|
232
232
|
input_tokens = getattr(usage, "prompt_tokens", 0) or 0
|
|
233
233
|
output_tokens = getattr(usage, "completion_tokens", 0) or 0
|
|
234
234
|
|
|
235
|
-
total_tokens = input_tokens + output_tokens
|
|
236
|
-
|
|
237
235
|
# Cost estimation
|
|
238
236
|
from evalview.core.pricing import calculate_cost
|
|
239
237
|
total_cost = calculate_cost(
|
|
@@ -26,7 +26,7 @@ import logging
|
|
|
26
26
|
import time
|
|
27
27
|
import uuid
|
|
28
28
|
from datetime import datetime, timezone
|
|
29
|
-
from typing import Any, Dict, List, Optional
|
|
29
|
+
from typing import Any, Dict, List, Optional
|
|
30
30
|
|
|
31
31
|
from evalview.adapters.base import AgentAdapter
|
|
32
32
|
from evalview.core.types import (
|
|
@@ -193,7 +193,6 @@ class PydanticAIAdapter(AgentAdapter):
|
|
|
193
193
|
usage = result.usage()
|
|
194
194
|
input_tokens = getattr(usage, "input_tokens", 0) or 0
|
|
195
195
|
output_tokens = getattr(usage, "output_tokens", 0) or 0
|
|
196
|
-
total_tokens = input_tokens + output_tokens
|
|
197
196
|
|
|
198
197
|
# Cost estimation (rough, based on common pricing)
|
|
199
198
|
from evalview.core.pricing import calculate_cost
|