aeval-framework 0.2.0__tar.gz → 0.3.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/.gitignore +13 -0
  2. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/PKG-INFO +2 -2
  3. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/pyproject.toml +10 -2
  4. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/__init__.py +1 -1
  5. aeval_framework-0.2.0/src/agent_eval/cli.py → aeval_framework-0.3.1/src/agent_eval/_cli_app.py +555 -130
  6. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/api/app.py +29 -2
  7. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/api/routes/runs.py +11 -36
  8. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/api/routes/tasks.py +2 -1
  9. aeval_framework-0.3.1/src/agent_eval/cli.py +103 -0
  10. aeval_framework-0.3.1/src/agent_eval/core/comparison.py +340 -0
  11. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/core/contract.py +281 -0
  12. aeval_framework-0.3.1/src/agent_eval/core/discovery.py +149 -0
  13. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/core/metrics.py +389 -0
  14. aeval_framework-0.3.1/src/agent_eval/core/packaging.py +442 -0
  15. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/core/runner.py +695 -43
  16. aeval_framework-0.3.1/src/agent_eval/core/simulator.py +144 -0
  17. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/core/types.py +679 -4
  18. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/graders/_evidence.py +5 -1
  19. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/graders/metric.py +28 -19
  20. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/graders/model_based.py +100 -12
  21. aeval_framework-0.3.1/src/agent_eval/graders/presentation_probes.py +392 -0
  22. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/graders/state_check.py +19 -2
  23. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/metrics/__init__.py +8 -0
  24. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/metrics/answer_relevancy.py +3 -9
  25. aeval_framework-0.3.1/src/agent_eval/metrics/base.py +352 -0
  26. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/metrics/batch_evaluation.py +12 -6
  27. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/metrics/context_precision.py +4 -10
  28. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/metrics/context_recall.py +4 -9
  29. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/metrics/faithfulness.py +4 -10
  30. aeval_framework-0.3.1/src/agent_eval/metrics/llm_judge.py +179 -0
  31. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/metrics/prompt_metric.py +6 -3
  32. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/metrics/pytest_plugin.py +106 -28
  33. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/metrics/report.py +89 -5
  34. aeval_framework-0.3.1/src/agent_eval/packs/__init__.py +8 -0
  35. aeval_framework-0.3.1/src/agent_eval/packs/starter/README.md +30 -0
  36. aeval_framework-0.3.1/src/agent_eval/packs/starter/manifest.json +8 -0
  37. aeval_framework-0.3.1/src/agent_eval/packs/starter/suite.yaml +29 -0
  38. aeval_framework-0.3.1/tests/smoke_achat_runner.py +94 -0
  39. aeval_framework-0.3.1/tests/test_agent_metric_catalog.py +474 -0
  40. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_api.py +44 -0
  41. aeval_framework-0.3.1/tests/test_baseline_comparison.py +250 -0
  42. aeval_framework-0.3.1/tests/test_cli.py +706 -0
  43. aeval_framework-0.3.1/tests/test_cli_entry_guard.py +193 -0
  44. aeval_framework-0.3.1/tests/test_conversation.py +772 -0
  45. aeval_framework-0.3.1/tests/test_discovery.py +267 -0
  46. aeval_framework-0.3.1/tests/test_environment_identity.py +339 -0
  47. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_eval_dataset_api.py +3 -3
  48. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_eval_metric_pipeline.py +6 -7
  49. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_eval_metrics_module.py +14 -14
  50. aeval_framework-0.3.1/tests/test_eval_pytest_plugin.py +540 -0
  51. aeval_framework-0.3.1/tests/test_holdout_and_canary.py +301 -0
  52. aeval_framework-0.3.1/tests/test_metric_roles_gates_agreement.py +734 -0
  53. aeval_framework-0.3.1/tests/test_packaging.py +436 -0
  54. aeval_framework-0.3.1/tests/test_power_analysis.py +133 -0
  55. aeval_framework-0.3.1/tests/test_presentation_probes.py +710 -0
  56. aeval_framework-0.3.1/tests/test_prompt_determinism.py +210 -0
  57. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_regrade_and_archive.py +249 -0
  58. aeval_framework-0.3.1/tests/test_version_single_source.py +131 -0
  59. aeval_framework-0.2.0/.coverage +0 -0
  60. aeval_framework-0.2.0/src/agent_eval/metrics/base.py +0 -212
  61. aeval_framework-0.2.0/src/agent_eval/metrics/llm_judge.py +0 -100
  62. aeval_framework-0.2.0/tests/test_cli.py +0 -361
  63. aeval_framework-0.2.0/tests/test_eval_pytest_plugin.py +0 -282
  64. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/README.md +0 -0
  65. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/api/__init__.py +0 -0
  66. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/api/events.py +0 -0
  67. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/api/routes/__init__.py +0 -0
  68. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/api/routes/datasets.py +0 -0
  69. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/api/routes/graders.py +0 -0
  70. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/api/routes/metrics.py +0 -0
  71. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/api/routes/suites.py +0 -0
  72. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/api/standalone.py +0 -0
  73. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/core/__init__.py +0 -0
  74. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/core/pricing.py +0 -0
  75. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/core/redaction.py +0 -0
  76. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/core/suite.py +0 -0
  77. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/dataset/__init__.py +0 -0
  78. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/dataset/models.py +0 -0
  79. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/dataset/quality.py +0 -0
  80. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/dataset/sources/__init__.py +0 -0
  81. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/dataset/sources/llm_generator.py +0 -0
  82. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/dataset/sources/manual.py +0 -0
  83. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/dataset/sources/regression.py +0 -0
  84. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/dataset/sources/trace_mining.py +0 -0
  85. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/dataset/storage.py +0 -0
  86. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/dataset/version.py +0 -0
  87. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/examples/__init__.py +0 -0
  88. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/examples/basic_usage.py +0 -0
  89. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/examples/mock_runner.py +0 -0
  90. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/graders/__init__.py +0 -0
  91. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/graders/_verdicts.py +0 -0
  92. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/graders/artifact_check.py +0 -0
  93. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/graders/code_based.py +0 -0
  94. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/graders/human.py +0 -0
  95. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/graders/step_level.py +0 -0
  96. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/graders/tool_calls.py +0 -0
  97. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/graders/transcript.py +0 -0
  98. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/metrics/synthetic_data.py +0 -0
  99. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/storage/__init__.py +0 -0
  100. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/storage/memory.py +0 -0
  101. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/storage/sqlite.py +0 -0
  102. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/trace/__init__.py +0 -0
  103. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/trace/mapping.py +0 -0
  104. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/trace/normalize.py +0 -0
  105. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/trace/observations.py +0 -0
  106. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/src/agent_eval/trace/phoenix.py +0 -0
  107. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/conftest.py +0 -0
  108. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_builtin_graders.py +0 -0
  109. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_collection_phases.py +0 -0
  110. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_e2e.py +0 -0
  111. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_eval_batch_evaluation.py +0 -0
  112. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_eval_dataset_sources.py +0 -0
  113. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_eval_dataset_storage.py +0 -0
  114. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_eval_metrics_api.py +0 -0
  115. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_eval_prompt_metric.py +0 -0
  116. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_eval_report.py +0 -0
  117. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_eval_task_history.py +0 -0
  118. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_evidence_capture_and_storage.py +0 -0
  119. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_graders.py +0 -0
  120. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_import_isolation.py +0 -0
  121. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_metrics.py +0 -0
  122. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_run_storage_fields.py +0 -0
  123. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_runner.py +0 -0
  124. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_sse.py +0 -0
  125. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_standalone_api.py +0 -0
  126. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_suite.py +0 -0
  127. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_termination_and_budgets.py +0 -0
  128. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_trace_normalization.py +0 -0
  129. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_trial_evidence.py +0 -0
  130. {aeval_framework-0.2.0 → aeval_framework-0.3.1}/tests/test_vocabulary_isolation.py +0 -0
@@ -9,6 +9,7 @@ build/
9
9
  .ruff_cache/
10
10
  *.db
11
11
  *.sqlite3
12
+ .coverage
12
13
 
13
14
  # Node / Next.js (dashboard)
14
15
  node_modules/
@@ -24,3 +25,15 @@ Thumbs.db
24
25
 
25
26
  # Editor-local state (Qoder)
26
27
  .qoder/
28
+
29
+ # AI tool state
30
+ .claude/
31
+ .codex/
32
+ .meituan-catpaw/
33
+
34
+ # Run artifacts & scratch notes
35
+ .live-run-output.txt
36
+ testlist.txt
37
+
38
+ # Personal (never commit)
39
+ docs/eval-harness-interview-prep.md
@@ -1,11 +1,11 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: aeval-framework
3
- Version: 0.2.0
3
+ Version: 0.3.1
4
4
  Summary: Aeval — open-source agent evaluation framework driven by OTel traces (suites, graders, metrics, storage, API, CLI)
5
5
  Project-URL: Homepage, https://github.com/QiYuyyds/Aeval
6
6
  Project-URL: Repository, https://github.com/QiYuyyds/Aeval
7
7
  Project-URL: Documentation, https://github.com/QiYuyyds/Aeval/tree/main/docs
8
- Project-URL: Changelog, https://github.com/QiYuyyds/Aeval/releases
8
+ Project-URL: Changelog, https://github.com/QiYuyyds/Aeval/blob/main/CHANGELOG.md
9
9
  Author: Aeval contributors
10
10
  License: MIT
11
11
  Keywords: agents,evaluation,llm,otel,testing,tracing
@@ -2,7 +2,9 @@
2
2
  # Distribution name: aeval-framework (PyPI). Python module: agent_eval, CLI: eval-suite.
3
3
  # (`agent-eval` on PyPI belongs to an unrelated project — UK AISI agenteval.)
4
4
  name = "aeval-framework"
5
- version = "0.2.0"
5
+ # 版本只有一个真相源: src/agent_eval/__init__.py 的 __version__ (design D1)。
6
+ # 手工写第二份 (旧的 pyproject `version =`) 才是缺陷 —— dynamic 让它无法被写出来。
7
+ dynamic = ["version"]
6
8
  description = "Aeval — open-source agent evaluation framework driven by OTel traces (suites, graders, metrics, storage, API, CLI)"
7
9
  readme = "README.md"
8
10
  requires-python = ">=3.11"
@@ -52,14 +54,20 @@ eval-suite = "agent_eval.cli:main"
52
54
  Homepage = "https://github.com/QiYuyyds/Aeval"
53
55
  Repository = "https://github.com/QiYuyyds/Aeval"
54
56
  Documentation = "https://github.com/QiYuyyds/Aeval/tree/main/docs"
55
- Changelog = "https://github.com/QiYuyyds/Aeval/releases"
57
+ Changelog = "https://github.com/QiYuyyds/Aeval/blob/main/CHANGELOG.md"
56
58
 
57
59
  [build-system]
58
60
  requires = ["hatchling"]
59
61
  build-backend = "hatchling.build"
60
62
 
63
+ [tool.hatch.version]
64
+ path = "src/agent_eval/__init__.py"
65
+
61
66
  [tool.hatch.build.targets.wheel]
62
67
  packages = ["src/agent_eval"]
68
+ # src/agent_eval/packs/** (含 starter pack 的 suite.yaml/manifest.json) 作为
69
+ # package data 随 wheel 分发 —— `eval-suite run demo` 的零 setup 首跑依赖它;
70
+ # 仓库 packs/starter/ 与之同源, tests/test_packaging.py 钉住两侧一致。
63
71
 
64
72
  [tool.pytest.ini_options]
65
73
  asyncio_mode = "auto"
@@ -11,4 +11,4 @@ Usage:
11
11
  result = await runner.run_suite(suite)
12
12
  """
13
13
 
14
- __version__ = "0.1.0"
14
+ __version__ = "0.3.1"