aeval-framework 0.2.0__tar.gz → 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/.gitignore +13 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/PKG-INFO +2 -2
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/pyproject.toml +10 -2
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/__init__.py +1 -1
- aeval_framework-0.2.0/src/agent_eval/cli.py → aeval_framework-0.3.1/src/agent_eval/_cli_app.py +555 -130
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/api/app.py +29 -2
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/api/routes/runs.py +11 -36
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/api/routes/tasks.py +2 -1
- aeval_framework-0.3.1/src/agent_eval/cli.py +103 -0
- aeval_framework-0.3.1/src/agent_eval/core/comparison.py +340 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/core/contract.py +281 -0
- aeval_framework-0.3.1/src/agent_eval/core/discovery.py +149 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/core/metrics.py +389 -0
- aeval_framework-0.3.1/src/agent_eval/core/packaging.py +442 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/core/runner.py +695 -43
- aeval_framework-0.3.1/src/agent_eval/core/simulator.py +144 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/core/types.py +679 -4
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/graders/_evidence.py +5 -1
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/graders/metric.py +28 -19
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/graders/model_based.py +100 -12
- aeval_framework-0.3.1/src/agent_eval/graders/presentation_probes.py +392 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/graders/state_check.py +19 -2
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/metrics/__init__.py +8 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/metrics/answer_relevancy.py +3 -9
- aeval_framework-0.3.1/src/agent_eval/metrics/base.py +352 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/metrics/batch_evaluation.py +12 -6
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/metrics/context_precision.py +4 -10
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/metrics/context_recall.py +4 -9
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/metrics/faithfulness.py +4 -10
- aeval_framework-0.3.1/src/agent_eval/metrics/llm_judge.py +179 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/metrics/prompt_metric.py +6 -3
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/metrics/pytest_plugin.py +106 -28
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/metrics/report.py +89 -5
- aeval_framework-0.3.1/src/agent_eval/packs/__init__.py +8 -0
- aeval_framework-0.3.1/src/agent_eval/packs/starter/README.md +30 -0
- aeval_framework-0.3.1/src/agent_eval/packs/starter/manifest.json +8 -0
- aeval_framework-0.3.1/src/agent_eval/packs/starter/suite.yaml +29 -0
- aeval_framework-0.3.1/tests/smoke_achat_runner.py +94 -0
- aeval_framework-0.3.1/tests/test_agent_metric_catalog.py +474 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_api.py +44 -0
- aeval_framework-0.3.1/tests/test_baseline_comparison.py +250 -0
- aeval_framework-0.3.1/tests/test_cli.py +706 -0
- aeval_framework-0.3.1/tests/test_cli_entry_guard.py +193 -0
- aeval_framework-0.3.1/tests/test_conversation.py +772 -0
- aeval_framework-0.3.1/tests/test_discovery.py +267 -0
- aeval_framework-0.3.1/tests/test_environment_identity.py +339 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_eval_dataset_api.py +3 -3
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_eval_metric_pipeline.py +6 -7
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_eval_metrics_module.py +14 -14
- aeval_framework-0.3.1/tests/test_eval_pytest_plugin.py +540 -0
- aeval_framework-0.3.1/tests/test_holdout_and_canary.py +301 -0
- aeval_framework-0.3.1/tests/test_metric_roles_gates_agreement.py +734 -0
- aeval_framework-0.3.1/tests/test_packaging.py +436 -0
- aeval_framework-0.3.1/tests/test_power_analysis.py +133 -0
- aeval_framework-0.3.1/tests/test_presentation_probes.py +710 -0
- aeval_framework-0.3.1/tests/test_prompt_determinism.py +210 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_regrade_and_archive.py +249 -0
- aeval_framework-0.3.1/tests/test_version_single_source.py +131 -0
- aeval_framework-0.2.0/.coverage +0 -0
- aeval_framework-0.2.0/src/agent_eval/metrics/base.py +0 -212
- aeval_framework-0.2.0/src/agent_eval/metrics/llm_judge.py +0 -100
- aeval_framework-0.2.0/tests/test_cli.py +0 -361
- aeval_framework-0.2.0/tests/test_eval_pytest_plugin.py +0 -282
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/README.md +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/api/__init__.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/api/events.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/api/routes/__init__.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/api/routes/datasets.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/api/routes/graders.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/api/routes/metrics.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/api/routes/suites.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/api/standalone.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/core/__init__.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/core/pricing.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/core/redaction.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/core/suite.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/dataset/__init__.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/dataset/models.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/dataset/quality.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/dataset/sources/__init__.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/dataset/sources/llm_generator.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/dataset/sources/manual.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/dataset/sources/regression.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/dataset/sources/trace_mining.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/dataset/storage.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/dataset/version.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/examples/__init__.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/examples/basic_usage.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/examples/mock_runner.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/graders/__init__.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/graders/_verdicts.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/graders/artifact_check.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/graders/code_based.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/graders/human.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/graders/step_level.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/graders/tool_calls.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/graders/transcript.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/metrics/synthetic_data.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/storage/__init__.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/storage/memory.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/storage/sqlite.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/trace/__init__.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/trace/mapping.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/trace/normalize.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/trace/observations.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/trace/phoenix.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/conftest.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_builtin_graders.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_collection_phases.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_e2e.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_eval_batch_evaluation.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_eval_dataset_sources.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_eval_dataset_storage.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_eval_metrics_api.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_eval_prompt_metric.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_eval_report.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_eval_task_history.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_evidence_capture_and_storage.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_graders.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_import_isolation.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_metrics.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_run_storage_fields.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_runner.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_sse.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_standalone_api.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_suite.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_termination_and_budgets.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_trace_normalization.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_trial_evidence.py +0 -0
- {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_vocabulary_isolation.py +0 -0
|
@@ -9,6 +9,7 @@ build/
|
|
|
9
9
|
.ruff_cache/
|
|
10
10
|
*.db
|
|
11
11
|
*.sqlite3
|
|
12
|
+
.coverage
|
|
12
13
|
|
|
13
14
|
# Node / Next.js (dashboard)
|
|
14
15
|
node_modules/
|
|
@@ -24,3 +25,15 @@ Thumbs.db
|
|
|
24
25
|
|
|
25
26
|
# Editor-local state (Qoder)
|
|
26
27
|
.qoder/
|
|
28
|
+
|
|
29
|
+
# AI tool state
|
|
30
|
+
.claude/
|
|
31
|
+
.codex/
|
|
32
|
+
.meituan-catpaw/
|
|
33
|
+
|
|
34
|
+
# Run artifacts & scratch notes
|
|
35
|
+
.live-run-output.txt
|
|
36
|
+
testlist.txt
|
|
37
|
+
|
|
38
|
+
# Personal (never commit)
|
|
39
|
+
docs/eval-harness-interview-prep.md
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: aeval-framework
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.1
|
|
4
4
|
Summary: Aeval — open-source agent evaluation framework driven by OTel traces (suites, graders, metrics, storage, API, CLI)
|
|
5
5
|
Project-URL: Homepage, https://github.com/QiYuyyds/Aeval
|
|
6
6
|
Project-URL: Repository, https://github.com/QiYuyyds/Aeval
|
|
7
7
|
Project-URL: Documentation, https://github.com/QiYuyyds/Aeval/tree/main/docs
|
|
8
|
-
Project-URL: Changelog, https://github.com/QiYuyyds/Aeval/
|
|
8
|
+
Project-URL: Changelog, https://github.com/QiYuyyds/Aeval/blob/main/CHANGELOG.md
|
|
9
9
|
Author: Aeval contributors
|
|
10
10
|
License: MIT
|
|
11
11
|
Keywords: agents,evaluation,llm,otel,testing,tracing
|
|
@@ -2,7 +2,9 @@
|
|
|
2
2
|
# Distribution name: aeval-framework (PyPI). Python module: agent_eval, CLI: eval-suite.
|
|
3
3
|
# (`agent-eval` on PyPI belongs to an unrelated project — UK AISI agenteval.)
|
|
4
4
|
name = "aeval-framework"
|
|
5
|
-
|
|
5
|
+
# 版本只有一个真相源: src/agent_eval/__init__.py 的 __version__ (design D1)。
|
|
6
|
+
# 手工写第二份 (旧的 pyproject `version =`) 才是缺陷 —— dynamic 让它无法被写出来。
|
|
7
|
+
dynamic = ["version"]
|
|
6
8
|
description = "Aeval — open-source agent evaluation framework driven by OTel traces (suites, graders, metrics, storage, API, CLI)"
|
|
7
9
|
readme = "README.md"
|
|
8
10
|
requires-python = ">=3.11"
|
|
@@ -52,14 +54,20 @@ eval-suite = "agent_eval.cli:main"
|
|
|
52
54
|
Homepage = "https://github.com/QiYuyyds/Aeval"
|
|
53
55
|
Repository = "https://github.com/QiYuyyds/Aeval"
|
|
54
56
|
Documentation = "https://github.com/QiYuyyds/Aeval/tree/main/docs"
|
|
55
|
-
Changelog = "https://github.com/QiYuyyds/Aeval/
|
|
57
|
+
Changelog = "https://github.com/QiYuyyds/Aeval/blob/main/CHANGELOG.md"
|
|
56
58
|
|
|
57
59
|
[build-system]
|
|
58
60
|
requires = ["hatchling"]
|
|
59
61
|
build-backend = "hatchling.build"
|
|
60
62
|
|
|
63
|
+
[tool.hatch.version]
|
|
64
|
+
path = "src/agent_eval/__init__.py"
|
|
65
|
+
|
|
61
66
|
[tool.hatch.build.targets.wheel]
|
|
62
67
|
packages = ["src/agent_eval"]
|
|
68
|
+
# src/agent_eval/packs/** (含 starter pack 的 suite.yaml/manifest.json) 作为
|
|
69
|
+
# package data 随 wheel 分发 —— `eval-suite run demo` 的零 setup 首跑依赖它;
|
|
70
|
+
# 仓库 packs/starter/ 与之同源, tests/test_packaging.py 钉住两侧一致。
|
|
63
71
|
|
|
64
72
|
[tool.pytest.ini_options]
|
|
65
73
|
asyncio_mode = "auto"
|