tool-eval-bench 2.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tool_eval_bench/__init__.py +41 -0
- tool_eval_bench/__main__.py +5 -0
- tool_eval_bench/_version.py +24 -0
- tool_eval_bench/adapters/__init__.py +0 -0
- tool_eval_bench/adapters/anthropic.py +744 -0
- tool_eval_bench/adapters/base.py +12 -0
- tool_eval_bench/adapters/factory.py +42 -0
- tool_eval_bench/adapters/gemini.py +750 -0
- tool_eval_bench/adapters/http_retry.py +506 -0
- tool_eval_bench/adapters/measurement.py +277 -0
- tool_eval_bench/adapters/openai_compat.py +667 -0
- tool_eval_bench/adapters/requests.py +113 -0
- tool_eval_bench/adapters/systemone.py +91 -0
- tool_eval_bench/adapters/wire_format.py +134 -0
- tool_eval_bench/api.py +278 -0
- tool_eval_bench/application/__init__.py +5 -0
- tool_eval_bench/application/decision_audit.py +291 -0
- tool_eval_bench/application/finalization.py +35 -0
- tool_eval_bench/application/mode_runs.py +71 -0
- tool_eval_bench/application/run_config.py +577 -0
- tool_eval_bench/application/run_context.py +144 -0
- tool_eval_bench/application/run_queries.py +77 -0
- tool_eval_bench/application/service.py +595 -0
- tool_eval_bench/cli/__init__.py +1 -0
- tool_eval_bench/cli/bench.py +37 -0
- tool_eval_bench/cli/command_registry.py +260 -0
- tool_eval_bench/cli/commands.py +108 -0
- tool_eval_bench/cli/compare_report.py +73 -0
- tool_eval_bench/cli/decision_live_display.py +490 -0
- tool_eval_bench/cli/dispatch.py +2331 -0
- tool_eval_bench/cli/display.py +707 -0
- tool_eval_bench/cli/headless.py +130 -0
- tool_eval_bench/cli/held_out.py +106 -0
- tool_eval_bench/cli/helpers.py +168 -0
- tool_eval_bench/cli/history.py +594 -0
- tool_eval_bench/cli/leaderboard.py +653 -0
- tool_eval_bench/cli/legacy_parser.py +725 -0
- tool_eval_bench/cli/local_commands.py +149 -0
- tool_eval_bench/cli/model_probe.py +402 -0
- tool_eval_bench/cli/modes.py +114 -0
- tool_eval_bench/cli/parser.py +281 -0
- tool_eval_bench/cli/perf.py +336 -0
- tool_eval_bench/cli/plugin_datasets.py +85 -0
- tool_eval_bench/cli/plugin_lifecycle.py +82 -0
- tool_eval_bench/cli/plugin_progress.py +197 -0
- tool_eval_bench/cli/plugin_runners.py +1202 -0
- tool_eval_bench/cli/pressure.py +518 -0
- tool_eval_bench/cli/probe.py +211 -0
- tool_eval_bench/cli/provider_env.py +99 -0
- tool_eval_bench/cli/resolve.py +25 -0
- tool_eval_bench/cli/run_io.py +225 -0
- tool_eval_bench/cli/scored_run.py +253 -0
- tool_eval_bench/cli/server.py +125 -0
- tool_eval_bench/cli/spec_bench.py +479 -0
- tool_eval_bench/cli/spec_live_display.py +1128 -0
- tool_eval_bench/cli/spec_live_rendering.py +384 -0
- tool_eval_bench/cli/timeout_advice.py +99 -0
- tool_eval_bench/compare_reports/__init__.py +1 -0
- tool_eval_bench/compare_reports/_common.py +133 -0
- tool_eval_bench/compare_reports/summary.py +978 -0
- tool_eval_bench/compare_reports/tool_eval.py +825 -0
- tool_eval_bench/domain/__init__.py +0 -0
- tool_eval_bench/domain/adapters.py +110 -0
- tool_eval_bench/domain/decision.py +208 -0
- tool_eval_bench/domain/engines.py +226 -0
- tool_eval_bench/domain/errors.py +55 -0
- tool_eval_bench/domain/filler.py +319 -0
- tool_eval_bench/domain/measurement.py +116 -0
- tool_eval_bench/domain/models.py +137 -0
- tool_eval_bench/domain/plugin.py +172 -0
- tool_eval_bench/domain/redaction.py +68 -0
- tool_eval_bench/domain/scenarios.py +725 -0
- tool_eval_bench/domain/spec_decode.py +68 -0
- tool_eval_bench/domain/tools.py +300 -0
- tool_eval_bench/domain/tools_large.py +703 -0
- tool_eval_bench/evals/__init__.py +0 -0
- tool_eval_bench/evals/helpers.py +1196 -0
- tool_eval_bench/evals/milestones.py +141 -0
- tool_eval_bench/evals/noise.py +381 -0
- tool_eval_bench/evals/packs.py +147 -0
- tool_eval_bench/evals/scenarios/__init__.py +86 -0
- tool_eval_bench/evals/scenarios/_registry.py +42 -0
- tool_eval_bench/evals/scenarios/adversarial/__init__.py +11 -0
- tool_eval_bench/evals/scenarios/adversarial/_shared.py +110 -0
- tool_eval_bench/evals/scenarios/adversarial/tc57.py +173 -0
- tool_eval_bench/evals/scenarios/adversarial/tc58.py +252 -0
- tool_eval_bench/evals/scenarios/adversarial/tc59.py +280 -0
- tool_eval_bench/evals/scenarios/adversarial/tc60.py +194 -0
- tool_eval_bench/evals/scenarios/agentic/__init__.py +15 -0
- tool_eval_bench/evals/scenarios/agentic/tc22.py +158 -0
- tool_eval_bench/evals/scenarios/agentic/tc23.py +92 -0
- tool_eval_bench/evals/scenarios/agentic/tc24.py +147 -0
- tool_eval_bench/evals/scenarios/agentic/tc25.py +145 -0
- tool_eval_bench/evals/scenarios/agentic/tc26.py +296 -0
- tool_eval_bench/evals/scenarios/agentic/tc27.py +156 -0
- tool_eval_bench/evals/scenarios/agentic/tc28.py +188 -0
- tool_eval_bench/evals/scenarios/agentic/tc29.py +212 -0
- tool_eval_bench/evals/scenarios/agentic/tc30.py +277 -0
- tool_eval_bench/evals/scenarios/agentic/tc31.py +152 -0
- tool_eval_bench/evals/scenarios/agentic/tc32.py +179 -0
- tool_eval_bench/evals/scenarios/agentic/tc33.py +296 -0
- tool_eval_bench/evals/scenarios/agentic/tc34.py +214 -0
- tool_eval_bench/evals/scenarios/agentic/tc35.py +152 -0
- tool_eval_bench/evals/scenarios/agentic/tc36.py +147 -0
- tool_eval_bench/evals/scenarios/agentic/tc41.py +129 -0
- tool_eval_bench/evals/scenarios/agentic/tc42.py +134 -0
- tool_eval_bench/evals/scenarios/agentic/tc43.py +115 -0
- tool_eval_bench/evals/scenarios/agentic/tc44.py +82 -0
- tool_eval_bench/evals/scenarios/agentic/tc45.py +129 -0
- tool_eval_bench/evals/scenarios/agentic/tc46.py +331 -0
- tool_eval_bench/evals/scenarios/agentic/tc47.py +195 -0
- tool_eval_bench/evals/scenarios/agentic/tc48.py +351 -0
- tool_eval_bench/evals/scenarios/agentic/tc49.py +307 -0
- tool_eval_bench/evals/scenarios/agentic/tc50.py +322 -0
- tool_eval_bench/evals/scenarios/core/__init__.py +16 -0
- tool_eval_bench/evals/scenarios/core/_shared.py +219 -0
- tool_eval_bench/evals/scenarios/core/tc01.py +149 -0
- tool_eval_bench/evals/scenarios/core/tc02.py +128 -0
- tool_eval_bench/evals/scenarios/core/tc03.py +236 -0
- tool_eval_bench/evals/scenarios/core/tc04.py +138 -0
- tool_eval_bench/evals/scenarios/core/tc05.py +135 -0
- tool_eval_bench/evals/scenarios/core/tc06.py +177 -0
- tool_eval_bench/evals/scenarios/core/tc07.py +252 -0
- tool_eval_bench/evals/scenarios/core/tc08.py +245 -0
- tool_eval_bench/evals/scenarios/core/tc09.py +147 -0
- tool_eval_bench/evals/scenarios/core/tc10.py +71 -0
- tool_eval_bench/evals/scenarios/core/tc11.py +71 -0
- tool_eval_bench/evals/scenarios/core/tc12.py +123 -0
- tool_eval_bench/evals/scenarios/core/tc13.py +232 -0
- tool_eval_bench/evals/scenarios/core/tc14.py +212 -0
- tool_eval_bench/evals/scenarios/core/tc15.py +182 -0
- tool_eval_bench/evals/scenarios/extended/__init__.py +12 -0
- tool_eval_bench/evals/scenarios/extended/_shared.py +101 -0
- tool_eval_bench/evals/scenarios/extended/tc16.py +233 -0
- tool_eval_bench/evals/scenarios/extended/tc17.py +154 -0
- tool_eval_bench/evals/scenarios/extended/tc18.py +211 -0
- tool_eval_bench/evals/scenarios/extended/tc19.py +176 -0
- tool_eval_bench/evals/scenarios/extended/tc20.py +204 -0
- tool_eval_bench/evals/scenarios/extended/tc21.py +191 -0
- tool_eval_bench/evals/scenarios/hardmode/__init__.py +22 -0
- tool_eval_bench/evals/scenarios/hardmode/_shared.py +8 -0
- tool_eval_bench/evals/scenarios/hardmode/tc70.py +194 -0
- tool_eval_bench/evals/scenarios/hardmode/tc71.py +273 -0
- tool_eval_bench/evals/scenarios/hardmode/tc72.py +232 -0
- tool_eval_bench/evals/scenarios/hardmode/tc73.py +294 -0
- tool_eval_bench/evals/scenarios/hardmode/tc74.py +338 -0
- tool_eval_bench/evals/scenarios/hardmode_documents/__init__.py +7 -0
- tool_eval_bench/evals/scenarios/hardmode_documents/_shared.py +33 -0
- tool_eval_bench/evals/scenarios/hardmode_documents/tc93.py +181 -0
- tool_eval_bench/evals/scenarios/hardmode_documents/tc94.py +147 -0
- tool_eval_bench/evals/scenarios/hardmode_documents/tc95.py +196 -0
- tool_eval_bench/evals/scenarios/hardmode_documents/tc96.py +159 -0
- tool_eval_bench/evals/scenarios/hardmode_documents/tc97.py +265 -0
- tool_eval_bench/evals/scenarios/hardmode_expanded/__init__.py +7 -0
- tool_eval_bench/evals/scenarios/hardmode_expanded/_shared.py +93 -0
- tool_eval_bench/evals/scenarios/hardmode_expanded/tc75.py +442 -0
- tool_eval_bench/evals/scenarios/hardmode_expanded/tc76.py +240 -0
- tool_eval_bench/evals/scenarios/hardmode_expanded/tc77.py +76 -0
- tool_eval_bench/evals/scenarios/hardmode_expanded/tc78.py +137 -0
- tool_eval_bench/evals/scenarios/hardmode_expanded/tc79.py +187 -0
- tool_eval_bench/evals/scenarios/hardmode_expanded/tc80.py +292 -0
- tool_eval_bench/evals/scenarios/hardmode_expanded/tc81.py +180 -0
- tool_eval_bench/evals/scenarios/hardmode_expanded/tc82.py +188 -0
- tool_eval_bench/evals/scenarios/hardmode_expanded/tc83.py +141 -0
- tool_eval_bench/evals/scenarios/hardmode_expanded/tc84.py +429 -0
- tool_eval_bench/evals/scenarios/hardmode_governance/__init__.py +7 -0
- tool_eval_bench/evals/scenarios/hardmode_governance/_shared.py +32 -0
- tool_eval_bench/evals/scenarios/hardmode_governance/tc90.py +348 -0
- tool_eval_bench/evals/scenarios/hardmode_governance/tc91.py +291 -0
- tool_eval_bench/evals/scenarios/hardmode_governance/tc92.py +244 -0
- tool_eval_bench/evals/scenarios/hardmode_transactional/__init__.py +7 -0
- tool_eval_bench/evals/scenarios/hardmode_transactional/_shared.py +75 -0
- tool_eval_bench/evals/scenarios/hardmode_transactional/tc85.py +519 -0
- tool_eval_bench/evals/scenarios/hardmode_transactional/tc86.py +479 -0
- tool_eval_bench/evals/scenarios/hardmode_transactional/tc87.py +430 -0
- tool_eval_bench/evals/scenarios/hardmode_transactional/tc88.py +85 -0
- tool_eval_bench/evals/scenarios/hardmode_transactional/tc89.py +416 -0
- tool_eval_bench/evals/scenarios/large_toolset/__init__.py +11 -0
- tool_eval_bench/evals/scenarios/large_toolset/_shared.py +31 -0
- tool_eval_bench/evals/scenarios/large_toolset/tc37.py +128 -0
- tool_eval_bench/evals/scenarios/large_toolset/tc38.py +243 -0
- tool_eval_bench/evals/scenarios/large_toolset/tc39.py +96 -0
- tool_eval_bench/evals/scenarios/large_toolset/tc40.py +234 -0
- tool_eval_bench/evals/scenarios/planning/__init__.py +17 -0
- tool_eval_bench/evals/scenarios/planning/_shared.py +143 -0
- tool_eval_bench/evals/scenarios/planning/tc51.py +307 -0
- tool_eval_bench/evals/scenarios/planning/tc52.py +177 -0
- tool_eval_bench/evals/scenarios/planning/tc53.py +301 -0
- tool_eval_bench/evals/scenarios/planning/tc54.py +196 -0
- tool_eval_bench/evals/scenarios/planning/tc55.py +219 -0
- tool_eval_bench/evals/scenarios/planning/tc56.py +211 -0
- tool_eval_bench/evals/scenarios/planning/tc61.py +361 -0
- tool_eval_bench/evals/scenarios/planning/tc62.py +516 -0
- tool_eval_bench/evals/scenarios/planning/tc63.py +307 -0
- tool_eval_bench/evals/scenarios/structured/__init__.py +16 -0
- tool_eval_bench/evals/scenarios/structured/_shared.py +64 -0
- tool_eval_bench/evals/scenarios/structured/tc64.py +140 -0
- tool_eval_bench/evals/scenarios/structured/tc65.py +160 -0
- tool_eval_bench/evals/scenarios/structured/tc66.py +210 -0
- tool_eval_bench/evals/scenarios/structured/tc67.py +191 -0
- tool_eval_bench/evals/scenarios/structured/tc68.py +164 -0
- tool_eval_bench/evals/scenarios/structured/tc69.py +273 -0
- tool_eval_bench/evals/variants.py +128 -0
- tool_eval_bench/evals/yaml_loader.py +505 -0
- tool_eval_bench/evals/yaml_scenarios/__init__.py +0 -0
- tool_eval_bench/evals/yaml_scenarios/chained_lookup.yaml +30 -0
- tool_eval_bench/evals/yaml_scenarios/no_tool_needed.yaml +15 -0
- tool_eval_bench/evals/yaml_scenarios/weather.yaml +20 -0
- tool_eval_bench/plugins/__init__.py +6 -0
- tool_eval_bench/plugins/decision/__init__.py +7 -0
- tool_eval_bench/plugins/decision/evaluator.py +73 -0
- tool_eval_bench/plugins/decision/live.py +309 -0
- tool_eval_bench/plugins/decision/metrics.py +102 -0
- tool_eval_bench/plugins/decision/plugin.py +337 -0
- tool_eval_bench/plugins/decision/render.py +195 -0
- tool_eval_bench/plugins/decision/typed_decisions.py +189 -0
- tool_eval_bench/plugins/decision/vendor/typed_decisions/LICENSE +202 -0
- tool_eval_bench/plugins/decision/vendor/typed_decisions/NOTICE +29 -0
- tool_eval_bench/plugins/decision/vendor/typed_decisions/manifest.json +24 -0
- tool_eval_bench/plugins/decision/vendor/typed_decisions/test.jsonl.gz +0 -0
- tool_eval_bench/plugins/gsm8k/__init__.py +1 -0
- tool_eval_bench/plugins/gsm8k/dataset.py +260 -0
- tool_eval_bench/plugins/gsm8k/evaluator.py +121 -0
- tool_eval_bench/plugins/gsm8k/plugin.py +471 -0
- tool_eval_bench/plugins/gsm8k/prompts.py +139 -0
- tool_eval_bench/plugins/hf_utils.py +333 -0
- tool_eval_bench/plugins/ifeval/__init__.py +1 -0
- tool_eval_bench/plugins/ifeval/checkers.py +644 -0
- tool_eval_bench/plugins/ifeval/dataset.py +174 -0
- tool_eval_bench/plugins/ifeval/evaluator.py +88 -0
- tool_eval_bench/plugins/ifeval/plugin.py +432 -0
- tool_eval_bench/plugins/mmlu/__init__.py +1 -0
- tool_eval_bench/plugins/mmlu/dataset.py +262 -0
- tool_eval_bench/plugins/mmlu/evaluator.py +111 -0
- tool_eval_bench/plugins/mmlu/plugin.py +533 -0
- tool_eval_bench/plugins/mmlu/prompts.py +78 -0
- tool_eval_bench/plugins/needle/__init__.py +19 -0
- tool_eval_bench/plugins/needle/haystack.py +201 -0
- tool_eval_bench/plugins/needle/plugin.py +335 -0
- tool_eval_bench/plugins/registry.py +40 -0
- tool_eval_bench/py.typed +0 -0
- tool_eval_bench/runner/__init__.py +0 -0
- tool_eval_bench/runner/context_pressure.py +777 -0
- tool_eval_bench/runner/llama_benchy.py +869 -0
- tool_eval_bench/runner/orchestrator.py +1483 -0
- tool_eval_bench/runner/service.py +20 -0
- tool_eval_bench/runner/spec_detection.py +361 -0
- tool_eval_bench/runner/spec_live.py +1019 -0
- tool_eval_bench/runner/speculative.py +816 -0
- tool_eval_bench/runner/throughput.py +1133 -0
- tool_eval_bench/schema.py +752 -0
- tool_eval_bench/storage/__init__.py +0 -0
- tool_eval_bench/storage/db.py +458 -0
- tool_eval_bench/storage/reports/__init__.py +173 -0
- tool_eval_bench/storage/reports/_common.py +342 -0
- tool_eval_bench/storage/reports/mode.py +74 -0
- tool_eval_bench/storage/reports/pressure.py +162 -0
- tool_eval_bench/storage/reports/scenario.py +426 -0
- tool_eval_bench/storage/reports/spec_decode.py +370 -0
- tool_eval_bench/storage/reports/summary.py +365 -0
- tool_eval_bench/storage/reports/throughput.py +88 -0
- tool_eval_bench/utils/__init__.py +0 -0
- tool_eval_bench/utils/fingerprint.py +77 -0
- tool_eval_bench/utils/headers.py +92 -0
- tool_eval_bench/utils/ids.py +30 -0
- tool_eval_bench/utils/metadata.py +1017 -0
- tool_eval_bench/utils/openai_compat.py +80 -0
- tool_eval_bench/utils/system_prompt.py +36 -0
- tool_eval_bench/utils/tokenizers.py +320 -0
- tool_eval_bench/utils/urls.py +236 -0
- tool_eval_bench-2.8.0.dist-info/METADATA +492 -0
- tool_eval_bench-2.8.0.dist-info/RECORD +276 -0
- tool_eval_bench-2.8.0.dist-info/WHEEL +5 -0
- tool_eval_bench-2.8.0.dist-info/entry_points.txt +2 -0
- tool_eval_bench-2.8.0.dist-info/licenses/LICENSE +21 -0
- tool_eval_bench-2.8.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""tool_eval_bench package."""
|
|
2
|
+
|
|
3
|
+
__all__ = ["__version__", "run_benchmark"]
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def _resolve_version() -> str:
|
|
7
|
+
"""Report the version of the code actually running.
|
|
8
|
+
|
|
9
|
+
``_version.py`` is generated by setuptools-scm at build time and is the most
|
|
10
|
+
precise source (it carries the commit for non-release builds). Installed
|
|
11
|
+
distribution metadata is the fallback, and a bare source tree with no build
|
|
12
|
+
step at all reports an explicitly unknown version rather than pretending to
|
|
13
|
+
be a release.
|
|
14
|
+
"""
|
|
15
|
+
try:
|
|
16
|
+
from tool_eval_bench._version import version
|
|
17
|
+
|
|
18
|
+
return str(version)
|
|
19
|
+
except ImportError:
|
|
20
|
+
pass
|
|
21
|
+
from importlib.metadata import PackageNotFoundError
|
|
22
|
+
from importlib.metadata import version as _dist_version
|
|
23
|
+
|
|
24
|
+
try:
|
|
25
|
+
return _dist_version("tool-eval-bench")
|
|
26
|
+
except PackageNotFoundError:
|
|
27
|
+
return "0.0.0+unknown"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
__version__ = _resolve_version()
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
async def run_benchmark(**kwargs):
|
|
34
|
+
"""Convenience re-export — see :func:`tool_eval_bench.api.run_benchmark`.
|
|
35
|
+
|
|
36
|
+
This is an async function; call it with ``await`` or wrap in
|
|
37
|
+
``asyncio.run(run_benchmark(...))``.
|
|
38
|
+
"""
|
|
39
|
+
from tool_eval_bench.api import run_benchmark as _run
|
|
40
|
+
|
|
41
|
+
return await _run(**kwargs)
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# file generated by vcs-versioning
|
|
2
|
+
# don't change, don't track in version control
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"__version__",
|
|
7
|
+
"__version_tuple__",
|
|
8
|
+
"version",
|
|
9
|
+
"version_tuple",
|
|
10
|
+
"__commit_id__",
|
|
11
|
+
"commit_id",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
version: str
|
|
15
|
+
__version__: str
|
|
16
|
+
__version_tuple__: tuple[int | str, ...]
|
|
17
|
+
version_tuple: tuple[int | str, ...]
|
|
18
|
+
commit_id: str | None
|
|
19
|
+
__commit_id__: str | None
|
|
20
|
+
|
|
21
|
+
__version__ = version = '2.8.0'
|
|
22
|
+
__version_tuple__ = version_tuple = (2, 8, 0)
|
|
23
|
+
|
|
24
|
+
__commit_id__ = commit_id = None
|
|
File without changes
|