tool-eval-bench 2.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (276) hide show
  1. tool_eval_bench/__init__.py +41 -0
  2. tool_eval_bench/__main__.py +5 -0
  3. tool_eval_bench/_version.py +24 -0
  4. tool_eval_bench/adapters/__init__.py +0 -0
  5. tool_eval_bench/adapters/anthropic.py +744 -0
  6. tool_eval_bench/adapters/base.py +12 -0
  7. tool_eval_bench/adapters/factory.py +42 -0
  8. tool_eval_bench/adapters/gemini.py +750 -0
  9. tool_eval_bench/adapters/http_retry.py +506 -0
  10. tool_eval_bench/adapters/measurement.py +277 -0
  11. tool_eval_bench/adapters/openai_compat.py +667 -0
  12. tool_eval_bench/adapters/requests.py +113 -0
  13. tool_eval_bench/adapters/systemone.py +91 -0
  14. tool_eval_bench/adapters/wire_format.py +134 -0
  15. tool_eval_bench/api.py +278 -0
  16. tool_eval_bench/application/__init__.py +5 -0
  17. tool_eval_bench/application/decision_audit.py +291 -0
  18. tool_eval_bench/application/finalization.py +35 -0
  19. tool_eval_bench/application/mode_runs.py +71 -0
  20. tool_eval_bench/application/run_config.py +577 -0
  21. tool_eval_bench/application/run_context.py +144 -0
  22. tool_eval_bench/application/run_queries.py +77 -0
  23. tool_eval_bench/application/service.py +595 -0
  24. tool_eval_bench/cli/__init__.py +1 -0
  25. tool_eval_bench/cli/bench.py +37 -0
  26. tool_eval_bench/cli/command_registry.py +260 -0
  27. tool_eval_bench/cli/commands.py +108 -0
  28. tool_eval_bench/cli/compare_report.py +73 -0
  29. tool_eval_bench/cli/decision_live_display.py +490 -0
  30. tool_eval_bench/cli/dispatch.py +2331 -0
  31. tool_eval_bench/cli/display.py +707 -0
  32. tool_eval_bench/cli/headless.py +130 -0
  33. tool_eval_bench/cli/held_out.py +106 -0
  34. tool_eval_bench/cli/helpers.py +168 -0
  35. tool_eval_bench/cli/history.py +594 -0
  36. tool_eval_bench/cli/leaderboard.py +653 -0
  37. tool_eval_bench/cli/legacy_parser.py +725 -0
  38. tool_eval_bench/cli/local_commands.py +149 -0
  39. tool_eval_bench/cli/model_probe.py +402 -0
  40. tool_eval_bench/cli/modes.py +114 -0
  41. tool_eval_bench/cli/parser.py +281 -0
  42. tool_eval_bench/cli/perf.py +336 -0
  43. tool_eval_bench/cli/plugin_datasets.py +85 -0
  44. tool_eval_bench/cli/plugin_lifecycle.py +82 -0
  45. tool_eval_bench/cli/plugin_progress.py +197 -0
  46. tool_eval_bench/cli/plugin_runners.py +1202 -0
  47. tool_eval_bench/cli/pressure.py +518 -0
  48. tool_eval_bench/cli/probe.py +211 -0
  49. tool_eval_bench/cli/provider_env.py +99 -0
  50. tool_eval_bench/cli/resolve.py +25 -0
  51. tool_eval_bench/cli/run_io.py +225 -0
  52. tool_eval_bench/cli/scored_run.py +253 -0
  53. tool_eval_bench/cli/server.py +125 -0
  54. tool_eval_bench/cli/spec_bench.py +479 -0
  55. tool_eval_bench/cli/spec_live_display.py +1128 -0
  56. tool_eval_bench/cli/spec_live_rendering.py +384 -0
  57. tool_eval_bench/cli/timeout_advice.py +99 -0
  58. tool_eval_bench/compare_reports/__init__.py +1 -0
  59. tool_eval_bench/compare_reports/_common.py +133 -0
  60. tool_eval_bench/compare_reports/summary.py +978 -0
  61. tool_eval_bench/compare_reports/tool_eval.py +825 -0
  62. tool_eval_bench/domain/__init__.py +0 -0
  63. tool_eval_bench/domain/adapters.py +110 -0
  64. tool_eval_bench/domain/decision.py +208 -0
  65. tool_eval_bench/domain/engines.py +226 -0
  66. tool_eval_bench/domain/errors.py +55 -0
  67. tool_eval_bench/domain/filler.py +319 -0
  68. tool_eval_bench/domain/measurement.py +116 -0
  69. tool_eval_bench/domain/models.py +137 -0
  70. tool_eval_bench/domain/plugin.py +172 -0
  71. tool_eval_bench/domain/redaction.py +68 -0
  72. tool_eval_bench/domain/scenarios.py +725 -0
  73. tool_eval_bench/domain/spec_decode.py +68 -0
  74. tool_eval_bench/domain/tools.py +300 -0
  75. tool_eval_bench/domain/tools_large.py +703 -0
  76. tool_eval_bench/evals/__init__.py +0 -0
  77. tool_eval_bench/evals/helpers.py +1196 -0
  78. tool_eval_bench/evals/milestones.py +141 -0
  79. tool_eval_bench/evals/noise.py +381 -0
  80. tool_eval_bench/evals/packs.py +147 -0
  81. tool_eval_bench/evals/scenarios/__init__.py +86 -0
  82. tool_eval_bench/evals/scenarios/_registry.py +42 -0
  83. tool_eval_bench/evals/scenarios/adversarial/__init__.py +11 -0
  84. tool_eval_bench/evals/scenarios/adversarial/_shared.py +110 -0
  85. tool_eval_bench/evals/scenarios/adversarial/tc57.py +173 -0
  86. tool_eval_bench/evals/scenarios/adversarial/tc58.py +252 -0
  87. tool_eval_bench/evals/scenarios/adversarial/tc59.py +280 -0
  88. tool_eval_bench/evals/scenarios/adversarial/tc60.py +194 -0
  89. tool_eval_bench/evals/scenarios/agentic/__init__.py +15 -0
  90. tool_eval_bench/evals/scenarios/agentic/tc22.py +158 -0
  91. tool_eval_bench/evals/scenarios/agentic/tc23.py +92 -0
  92. tool_eval_bench/evals/scenarios/agentic/tc24.py +147 -0
  93. tool_eval_bench/evals/scenarios/agentic/tc25.py +145 -0
  94. tool_eval_bench/evals/scenarios/agentic/tc26.py +296 -0
  95. tool_eval_bench/evals/scenarios/agentic/tc27.py +156 -0
  96. tool_eval_bench/evals/scenarios/agentic/tc28.py +188 -0
  97. tool_eval_bench/evals/scenarios/agentic/tc29.py +212 -0
  98. tool_eval_bench/evals/scenarios/agentic/tc30.py +277 -0
  99. tool_eval_bench/evals/scenarios/agentic/tc31.py +152 -0
  100. tool_eval_bench/evals/scenarios/agentic/tc32.py +179 -0
  101. tool_eval_bench/evals/scenarios/agentic/tc33.py +296 -0
  102. tool_eval_bench/evals/scenarios/agentic/tc34.py +214 -0
  103. tool_eval_bench/evals/scenarios/agentic/tc35.py +152 -0
  104. tool_eval_bench/evals/scenarios/agentic/tc36.py +147 -0
  105. tool_eval_bench/evals/scenarios/agentic/tc41.py +129 -0
  106. tool_eval_bench/evals/scenarios/agentic/tc42.py +134 -0
  107. tool_eval_bench/evals/scenarios/agentic/tc43.py +115 -0
  108. tool_eval_bench/evals/scenarios/agentic/tc44.py +82 -0
  109. tool_eval_bench/evals/scenarios/agentic/tc45.py +129 -0
  110. tool_eval_bench/evals/scenarios/agentic/tc46.py +331 -0
  111. tool_eval_bench/evals/scenarios/agentic/tc47.py +195 -0
  112. tool_eval_bench/evals/scenarios/agentic/tc48.py +351 -0
  113. tool_eval_bench/evals/scenarios/agentic/tc49.py +307 -0
  114. tool_eval_bench/evals/scenarios/agentic/tc50.py +322 -0
  115. tool_eval_bench/evals/scenarios/core/__init__.py +16 -0
  116. tool_eval_bench/evals/scenarios/core/_shared.py +219 -0
  117. tool_eval_bench/evals/scenarios/core/tc01.py +149 -0
  118. tool_eval_bench/evals/scenarios/core/tc02.py +128 -0
  119. tool_eval_bench/evals/scenarios/core/tc03.py +236 -0
  120. tool_eval_bench/evals/scenarios/core/tc04.py +138 -0
  121. tool_eval_bench/evals/scenarios/core/tc05.py +135 -0
  122. tool_eval_bench/evals/scenarios/core/tc06.py +177 -0
  123. tool_eval_bench/evals/scenarios/core/tc07.py +252 -0
  124. tool_eval_bench/evals/scenarios/core/tc08.py +245 -0
  125. tool_eval_bench/evals/scenarios/core/tc09.py +147 -0
  126. tool_eval_bench/evals/scenarios/core/tc10.py +71 -0
  127. tool_eval_bench/evals/scenarios/core/tc11.py +71 -0
  128. tool_eval_bench/evals/scenarios/core/tc12.py +123 -0
  129. tool_eval_bench/evals/scenarios/core/tc13.py +232 -0
  130. tool_eval_bench/evals/scenarios/core/tc14.py +212 -0
  131. tool_eval_bench/evals/scenarios/core/tc15.py +182 -0
  132. tool_eval_bench/evals/scenarios/extended/__init__.py +12 -0
  133. tool_eval_bench/evals/scenarios/extended/_shared.py +101 -0
  134. tool_eval_bench/evals/scenarios/extended/tc16.py +233 -0
  135. tool_eval_bench/evals/scenarios/extended/tc17.py +154 -0
  136. tool_eval_bench/evals/scenarios/extended/tc18.py +211 -0
  137. tool_eval_bench/evals/scenarios/extended/tc19.py +176 -0
  138. tool_eval_bench/evals/scenarios/extended/tc20.py +204 -0
  139. tool_eval_bench/evals/scenarios/extended/tc21.py +191 -0
  140. tool_eval_bench/evals/scenarios/hardmode/__init__.py +22 -0
  141. tool_eval_bench/evals/scenarios/hardmode/_shared.py +8 -0
  142. tool_eval_bench/evals/scenarios/hardmode/tc70.py +194 -0
  143. tool_eval_bench/evals/scenarios/hardmode/tc71.py +273 -0
  144. tool_eval_bench/evals/scenarios/hardmode/tc72.py +232 -0
  145. tool_eval_bench/evals/scenarios/hardmode/tc73.py +294 -0
  146. tool_eval_bench/evals/scenarios/hardmode/tc74.py +338 -0
  147. tool_eval_bench/evals/scenarios/hardmode_documents/__init__.py +7 -0
  148. tool_eval_bench/evals/scenarios/hardmode_documents/_shared.py +33 -0
  149. tool_eval_bench/evals/scenarios/hardmode_documents/tc93.py +181 -0
  150. tool_eval_bench/evals/scenarios/hardmode_documents/tc94.py +147 -0
  151. tool_eval_bench/evals/scenarios/hardmode_documents/tc95.py +196 -0
  152. tool_eval_bench/evals/scenarios/hardmode_documents/tc96.py +159 -0
  153. tool_eval_bench/evals/scenarios/hardmode_documents/tc97.py +265 -0
  154. tool_eval_bench/evals/scenarios/hardmode_expanded/__init__.py +7 -0
  155. tool_eval_bench/evals/scenarios/hardmode_expanded/_shared.py +93 -0
  156. tool_eval_bench/evals/scenarios/hardmode_expanded/tc75.py +442 -0
  157. tool_eval_bench/evals/scenarios/hardmode_expanded/tc76.py +240 -0
  158. tool_eval_bench/evals/scenarios/hardmode_expanded/tc77.py +76 -0
  159. tool_eval_bench/evals/scenarios/hardmode_expanded/tc78.py +137 -0
  160. tool_eval_bench/evals/scenarios/hardmode_expanded/tc79.py +187 -0
  161. tool_eval_bench/evals/scenarios/hardmode_expanded/tc80.py +292 -0
  162. tool_eval_bench/evals/scenarios/hardmode_expanded/tc81.py +180 -0
  163. tool_eval_bench/evals/scenarios/hardmode_expanded/tc82.py +188 -0
  164. tool_eval_bench/evals/scenarios/hardmode_expanded/tc83.py +141 -0
  165. tool_eval_bench/evals/scenarios/hardmode_expanded/tc84.py +429 -0
  166. tool_eval_bench/evals/scenarios/hardmode_governance/__init__.py +7 -0
  167. tool_eval_bench/evals/scenarios/hardmode_governance/_shared.py +32 -0
  168. tool_eval_bench/evals/scenarios/hardmode_governance/tc90.py +348 -0
  169. tool_eval_bench/evals/scenarios/hardmode_governance/tc91.py +291 -0
  170. tool_eval_bench/evals/scenarios/hardmode_governance/tc92.py +244 -0
  171. tool_eval_bench/evals/scenarios/hardmode_transactional/__init__.py +7 -0
  172. tool_eval_bench/evals/scenarios/hardmode_transactional/_shared.py +75 -0
  173. tool_eval_bench/evals/scenarios/hardmode_transactional/tc85.py +519 -0
  174. tool_eval_bench/evals/scenarios/hardmode_transactional/tc86.py +479 -0
  175. tool_eval_bench/evals/scenarios/hardmode_transactional/tc87.py +430 -0
  176. tool_eval_bench/evals/scenarios/hardmode_transactional/tc88.py +85 -0
  177. tool_eval_bench/evals/scenarios/hardmode_transactional/tc89.py +416 -0
  178. tool_eval_bench/evals/scenarios/large_toolset/__init__.py +11 -0
  179. tool_eval_bench/evals/scenarios/large_toolset/_shared.py +31 -0
  180. tool_eval_bench/evals/scenarios/large_toolset/tc37.py +128 -0
  181. tool_eval_bench/evals/scenarios/large_toolset/tc38.py +243 -0
  182. tool_eval_bench/evals/scenarios/large_toolset/tc39.py +96 -0
  183. tool_eval_bench/evals/scenarios/large_toolset/tc40.py +234 -0
  184. tool_eval_bench/evals/scenarios/planning/__init__.py +17 -0
  185. tool_eval_bench/evals/scenarios/planning/_shared.py +143 -0
  186. tool_eval_bench/evals/scenarios/planning/tc51.py +307 -0
  187. tool_eval_bench/evals/scenarios/planning/tc52.py +177 -0
  188. tool_eval_bench/evals/scenarios/planning/tc53.py +301 -0
  189. tool_eval_bench/evals/scenarios/planning/tc54.py +196 -0
  190. tool_eval_bench/evals/scenarios/planning/tc55.py +219 -0
  191. tool_eval_bench/evals/scenarios/planning/tc56.py +211 -0
  192. tool_eval_bench/evals/scenarios/planning/tc61.py +361 -0
  193. tool_eval_bench/evals/scenarios/planning/tc62.py +516 -0
  194. tool_eval_bench/evals/scenarios/planning/tc63.py +307 -0
  195. tool_eval_bench/evals/scenarios/structured/__init__.py +16 -0
  196. tool_eval_bench/evals/scenarios/structured/_shared.py +64 -0
  197. tool_eval_bench/evals/scenarios/structured/tc64.py +140 -0
  198. tool_eval_bench/evals/scenarios/structured/tc65.py +160 -0
  199. tool_eval_bench/evals/scenarios/structured/tc66.py +210 -0
  200. tool_eval_bench/evals/scenarios/structured/tc67.py +191 -0
  201. tool_eval_bench/evals/scenarios/structured/tc68.py +164 -0
  202. tool_eval_bench/evals/scenarios/structured/tc69.py +273 -0
  203. tool_eval_bench/evals/variants.py +128 -0
  204. tool_eval_bench/evals/yaml_loader.py +505 -0
  205. tool_eval_bench/evals/yaml_scenarios/__init__.py +0 -0
  206. tool_eval_bench/evals/yaml_scenarios/chained_lookup.yaml +30 -0
  207. tool_eval_bench/evals/yaml_scenarios/no_tool_needed.yaml +15 -0
  208. tool_eval_bench/evals/yaml_scenarios/weather.yaml +20 -0
  209. tool_eval_bench/plugins/__init__.py +6 -0
  210. tool_eval_bench/plugins/decision/__init__.py +7 -0
  211. tool_eval_bench/plugins/decision/evaluator.py +73 -0
  212. tool_eval_bench/plugins/decision/live.py +309 -0
  213. tool_eval_bench/plugins/decision/metrics.py +102 -0
  214. tool_eval_bench/plugins/decision/plugin.py +337 -0
  215. tool_eval_bench/plugins/decision/render.py +195 -0
  216. tool_eval_bench/plugins/decision/typed_decisions.py +189 -0
  217. tool_eval_bench/plugins/decision/vendor/typed_decisions/LICENSE +202 -0
  218. tool_eval_bench/plugins/decision/vendor/typed_decisions/NOTICE +29 -0
  219. tool_eval_bench/plugins/decision/vendor/typed_decisions/manifest.json +24 -0
  220. tool_eval_bench/plugins/decision/vendor/typed_decisions/test.jsonl.gz +0 -0
  221. tool_eval_bench/plugins/gsm8k/__init__.py +1 -0
  222. tool_eval_bench/plugins/gsm8k/dataset.py +260 -0
  223. tool_eval_bench/plugins/gsm8k/evaluator.py +121 -0
  224. tool_eval_bench/plugins/gsm8k/plugin.py +471 -0
  225. tool_eval_bench/plugins/gsm8k/prompts.py +139 -0
  226. tool_eval_bench/plugins/hf_utils.py +333 -0
  227. tool_eval_bench/plugins/ifeval/__init__.py +1 -0
  228. tool_eval_bench/plugins/ifeval/checkers.py +644 -0
  229. tool_eval_bench/plugins/ifeval/dataset.py +174 -0
  230. tool_eval_bench/plugins/ifeval/evaluator.py +88 -0
  231. tool_eval_bench/plugins/ifeval/plugin.py +432 -0
  232. tool_eval_bench/plugins/mmlu/__init__.py +1 -0
  233. tool_eval_bench/plugins/mmlu/dataset.py +262 -0
  234. tool_eval_bench/plugins/mmlu/evaluator.py +111 -0
  235. tool_eval_bench/plugins/mmlu/plugin.py +533 -0
  236. tool_eval_bench/plugins/mmlu/prompts.py +78 -0
  237. tool_eval_bench/plugins/needle/__init__.py +19 -0
  238. tool_eval_bench/plugins/needle/haystack.py +201 -0
  239. tool_eval_bench/plugins/needle/plugin.py +335 -0
  240. tool_eval_bench/plugins/registry.py +40 -0
  241. tool_eval_bench/py.typed +0 -0
  242. tool_eval_bench/runner/__init__.py +0 -0
  243. tool_eval_bench/runner/context_pressure.py +777 -0
  244. tool_eval_bench/runner/llama_benchy.py +869 -0
  245. tool_eval_bench/runner/orchestrator.py +1483 -0
  246. tool_eval_bench/runner/service.py +20 -0
  247. tool_eval_bench/runner/spec_detection.py +361 -0
  248. tool_eval_bench/runner/spec_live.py +1019 -0
  249. tool_eval_bench/runner/speculative.py +816 -0
  250. tool_eval_bench/runner/throughput.py +1133 -0
  251. tool_eval_bench/schema.py +752 -0
  252. tool_eval_bench/storage/__init__.py +0 -0
  253. tool_eval_bench/storage/db.py +458 -0
  254. tool_eval_bench/storage/reports/__init__.py +173 -0
  255. tool_eval_bench/storage/reports/_common.py +342 -0
  256. tool_eval_bench/storage/reports/mode.py +74 -0
  257. tool_eval_bench/storage/reports/pressure.py +162 -0
  258. tool_eval_bench/storage/reports/scenario.py +426 -0
  259. tool_eval_bench/storage/reports/spec_decode.py +370 -0
  260. tool_eval_bench/storage/reports/summary.py +365 -0
  261. tool_eval_bench/storage/reports/throughput.py +88 -0
  262. tool_eval_bench/utils/__init__.py +0 -0
  263. tool_eval_bench/utils/fingerprint.py +77 -0
  264. tool_eval_bench/utils/headers.py +92 -0
  265. tool_eval_bench/utils/ids.py +30 -0
  266. tool_eval_bench/utils/metadata.py +1017 -0
  267. tool_eval_bench/utils/openai_compat.py +80 -0
  268. tool_eval_bench/utils/system_prompt.py +36 -0
  269. tool_eval_bench/utils/tokenizers.py +320 -0
  270. tool_eval_bench/utils/urls.py +236 -0
  271. tool_eval_bench-2.8.0.dist-info/METADATA +492 -0
  272. tool_eval_bench-2.8.0.dist-info/RECORD +276 -0
  273. tool_eval_bench-2.8.0.dist-info/WHEEL +5 -0
  274. tool_eval_bench-2.8.0.dist-info/entry_points.txt +2 -0
  275. tool_eval_bench-2.8.0.dist-info/licenses/LICENSE +21 -0
  276. tool_eval_bench-2.8.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,41 @@
1
+ """tool_eval_bench package."""
2
+
3
+ __all__ = ["__version__", "run_benchmark"]
4
+
5
+
6
+ def _resolve_version() -> str:
7
+ """Report the version of the code actually running.
8
+
9
+ ``_version.py`` is generated by setuptools-scm at build time and is the most
10
+ precise source (it carries the commit for non-release builds). Installed
11
+ distribution metadata is the fallback, and a bare source tree with no build
12
+ step at all reports an explicitly unknown version rather than pretending to
13
+ be a release.
14
+ """
15
+ try:
16
+ from tool_eval_bench._version import version
17
+
18
+ return str(version)
19
+ except ImportError:
20
+ pass
21
+ from importlib.metadata import PackageNotFoundError
22
+ from importlib.metadata import version as _dist_version
23
+
24
+ try:
25
+ return _dist_version("tool-eval-bench")
26
+ except PackageNotFoundError:
27
+ return "0.0.0+unknown"
28
+
29
+
30
+ __version__ = _resolve_version()
31
+
32
+
33
+ async def run_benchmark(**kwargs):
34
+ """Convenience re-export — see :func:`tool_eval_bench.api.run_benchmark`.
35
+
36
+ This is an async function; call it with ``await`` or wrap in
37
+ ``asyncio.run(run_benchmark(...))``.
38
+ """
39
+ from tool_eval_bench.api import run_benchmark as _run
40
+
41
+ return await _run(**kwargs)
@@ -0,0 +1,5 @@
1
+ """Allow running tool-eval-bench as ``python -m tool_eval_bench``."""
2
+
3
+ from tool_eval_bench.cli.dispatch import main
4
+
5
+ main()
@@ -0,0 +1,24 @@
1
+ # file generated by vcs-versioning
2
+ # don't change, don't track in version control
3
+ from __future__ import annotations
4
+
5
+ __all__ = [
6
+ "__version__",
7
+ "__version_tuple__",
8
+ "version",
9
+ "version_tuple",
10
+ "__commit_id__",
11
+ "commit_id",
12
+ ]
13
+
14
+ version: str
15
+ __version__: str
16
+ __version_tuple__: tuple[int | str, ...]
17
+ version_tuple: tuple[int | str, ...]
18
+ commit_id: str | None
19
+ __commit_id__: str | None
20
+
21
+ __version__ = version = '2.8.0'
22
+ __version_tuple__ = version_tuple = (2, 8, 0)
23
+
24
+ __commit_id__ = commit_id = None
File without changes