onetool-mcp 1.0.0b1__py3-none-any.whl

bench/__init__.py

@@ -0,0 +1,5 @@
+"""OneTool benchmark harness CLI package.
+
+Entry point for the `bench` command used for benchmarking
+agent + MCP configurations.
+"""

bench/cli.py

@@ -0,0 +1,69 @@
+"""Bench CLI entry point for OneTool benchmark harness."""
+
+from __future__ import annotations
+
+import typer
+
+import ot
+from ot._cli import create_cli, version_callback
+
+app = create_cli(
+    "bench",
+    "OneTool benchmark harness for benchmarking agent + MCP configurations.",
+    no_args_is_help=True,
+)
+
+
+@app.callback()
+def main(
+    _version: bool | None = typer.Option(
+        None,
+        "--version",
+        "-v",
+        callback=version_callback("bench", ot.__version__),
+        is_eager=True,
+        help="Show version and exit.",
+    ),
+) -> None:
+    """OneTool benchmark harness for benchmarking agent + MCP configurations.
+
+    Commands:
+        run     - Run tasks (direct MCP calls or agent benchmarks)
+
+    Task types:
+        type: direct  - Direct MCP tool invocation (no LLM)
+        type: harness - LLM benchmark with MCP servers (default)
+
+    External client testing:
+        Use `just client` to test with OpenCode or Claude Code.
+    """
+    import sys
+
+    # Allow --help without requiring global config
+    if any(arg in sys.argv for arg in ("--help", "-h")):
+        return
+
+    # Require global config directory (created by onetool)
+    from ot.paths import get_global_dir
+
+    global_dir = get_global_dir()
+    if not global_dir.exists():
+        print(
+            f"Error: {global_dir} not found.\n"
+            "Run 'onetool init' to initialize OneTool configuration.",
+            file=sys.stderr,
+        )
+        raise typer.Exit(1)
+
+
+# Import subcommands to register them
+from bench import run  # noqa: E402, F401
+
+
+def cli() -> None:
+    """Run the CLI application."""
+    app()
+
+
+if __name__ == "__main__":
+    cli()

bench/harness/__init__.py

@@ -0,0 +1,66 @@
+"""Development harness for benchmarking OneTool.
+
+This module provides utilities for running LLM prompts with MCP servers,
+collecting metrics, and comparing results across different configurations.
+"""
+
+from bench.harness.client import (
+    DEFAULT_TIMEOUT,
+    MCPConnection,
+    call_tool,
+    mcp_tools_to_openai,
+    multi_server_tools_to_openai,
+)
+from bench.harness.config import (
+    DefaultsConfig,
+    EvaluateConfig,
+    HarnessConfig,
+    ScenarioConfig,
+    ServerConfig,
+    TaskConfig,
+    load_config,
+    load_harness_config,
+)
+from bench.harness.csv_writer import write_results_csv
+from bench.harness.evaluate import (
+    evaluate_deterministic,
+    evaluate_regex,
+    evaluate_task,
+    resolve_evaluator,
+)
+from bench.harness.metrics import (
+    EvaluationResult,
+    LLMCallMetrics,
+    ScenarioResult,
+    TaskResult,
+    calculate_cost,
+)
+from bench.harness.runner import AgenticRunner, split_prompts
+
+__all__ = [
+    "DEFAULT_TIMEOUT",
+    "AgenticRunner",
+    "DefaultsConfig",
+    "EvaluateConfig",
+    "EvaluationResult",
+    "HarnessConfig",
+    "LLMCallMetrics",
+    "MCPConnection",
+    "ScenarioConfig",
+    "ScenarioResult",
+    "ServerConfig",
+    "TaskConfig",
+    "TaskResult",
+    "calculate_cost",
+    "call_tool",
+    "evaluate_deterministic",
+    "evaluate_regex",
+    "evaluate_task",
+    "load_config",
+    "load_harness_config",
+    "mcp_tools_to_openai",
+    "multi_server_tools_to_openai",
+    "resolve_evaluator",
+    "split_prompts",
+    "write_results_csv",
+]