mcpbr 0.5.0__py3-none-any.whl → 0.6.0__py3-none-any.whl

mcpbr/providers.py

@@ -1,6 +1,8 @@
 """Model provider abstractions for different LLM APIs."""
 
+import json
 import os
+import uuid
 from dataclasses import dataclass, field
 from typing import Any, Protocol, runtime_checkable
 
@@ -142,8 +144,6 @@ class AnthropicProvider:
             if block.type == "text":
                 content_text = block.text
             elif block.type == "tool_use":
-                import json
-
                 tool_calls.append(
                     ToolCall(
                         id=block.id,
@@ -170,8 +170,321 @@ class AnthropicProvider:
         )
 
 
+class OpenAIProvider:
+    """Provider for OpenAI API (GPT models)."""
+
+    def __init__(
+        self,
+        model: str,
+        api_key: str | None = None,
+    ) -> None:
+        """Initialize OpenAI provider.
+
+        Args:
+            model: OpenAI model ID (e.g., 'gpt-4o', 'gpt-4-turbo').
+            api_key: API key. If None, uses OPENAI_API_KEY env var.
+        """
+        self._model = model
+        self._api_key = api_key or os.environ.get("OPENAI_API_KEY")
+        if not self._api_key:
+            raise ValueError(
+                "OpenAI API key required. Set OPENAI_API_KEY environment variable "
+                "or pass api_key parameter."
+            )
+        import openai
+
+        self._client = openai.OpenAI(api_key=self._api_key)
+
+    @property
+    def model(self) -> str:
+        return self._model
+
+    def get_tool_format(self) -> str:
+        return "openai"
+
+    def chat(
+        self,
+        messages: list[dict[str, Any]],
+        tools: list[dict[str, Any]] | None = None,
+        max_tokens: int = 4096,
+    ) -> ChatResponse:
+        kwargs: dict[str, Any] = {
+            "model": self._model,
+            "messages": messages,
+            "max_tokens": max_tokens,
+        }
+        if tools:
+            kwargs["tools"] = tools
+
+        response = self._client.chat.completions.create(**kwargs)
+
+        if not response.choices:
+            raise RuntimeError("OpenAI API returned empty response choices")
+
+        choice = response.choices[0]
+        tool_calls = []
+        if choice.message.tool_calls:
+            for tc in choice.message.tool_calls:
+                tool_calls.append(
+                    ToolCall(
+                        id=tc.id,
+                        name=tc.function.name,
+                        arguments=tc.function.arguments,
+                    )
+                )
+
+        return ChatResponse(
+            message=ChatMessage(
+                role="assistant",
+                content=choice.message.content,
+                tool_calls=tool_calls,
+            ),
+            finish_reason=choice.finish_reason,
+            input_tokens=response.usage.prompt_tokens,
+            output_tokens=response.usage.completion_tokens,
+        )
+
+
+class GeminiProvider:
+    """Provider for Google Gemini API."""
+
+    def __init__(
+        self,
+        model: str,
+        api_key: str | None = None,
+    ) -> None:
+        """Initialize Gemini provider.
+
+        Args:
+            model: Gemini model ID (e.g., 'gemini-2.0-flash', 'gemini-1.5-pro').
+            api_key: API key. If None, uses GOOGLE_API_KEY env var.
+        """
+        self._model = model
+        self._api_key = api_key or os.environ.get("GOOGLE_API_KEY")
+        if not self._api_key:
+            raise ValueError(
+                "Google API key required. Set GOOGLE_API_KEY environment variable "
+                "or pass api_key parameter."
+            )
+        import google.generativeai as genai
+
+        genai.configure(api_key=self._api_key)
+        self._genai = genai
+        self._client = genai.GenerativeModel(model)
+
+    @property
+    def model(self) -> str:
+        return self._model
+
+    def get_tool_format(self) -> str:
+        return "openai"
+
+    def _convert_messages(
+        self, messages: list[dict[str, Any]]
+    ) -> tuple[list[dict[str, Any]], str | None]:
+        """Convert OpenAI-style messages to Gemini content format.
+
+        Extracts system messages to use as system_instruction (Gemini's native
+        system prompt support), and converts the remaining messages.
+
+        Args:
+            messages: List of OpenAI-style message dicts.
+
+        Returns:
+            Tuple of (contents, system_instruction). system_instruction is None
+            if no system message was found.
+        """
+        contents: list[dict[str, Any]] = []
+        system_instruction: str | None = None
+        for msg in messages:
+            role = msg.get("role", "user")
+            if role == "system":
+                system_instruction = msg.get("content", "")
+            elif role == "assistant":
+                contents.append({"role": "model", "parts": [msg.get("content", "")]})
+            else:
+                contents.append({"role": role, "parts": [msg.get("content", "")]})
+        return contents, system_instruction
+
+    def _convert_tools(self, tools: list[dict[str, Any]] | None) -> list[Any] | None:
+        """Convert OpenAI-style tool definitions to Gemini function declarations.
+
+        Args:
+            tools: List of OpenAI-style tool dicts.
+
+        Returns:
+            List of Gemini Tool objects, or None.
+        """
+        if not tools:
+            return None
+
+        function_declarations = []
+        for tool in tools:
+            func = tool.get("function", {})
+            function_declarations.append(
+                self._genai.protos.FunctionDeclaration(
+                    name=func.get("name", ""),
+                    description=func.get("description", ""),
+                    parameters=func.get("parameters"),
+                )
+            )
+        return [self._genai.protos.Tool(function_declarations=function_declarations)]
+
+    def chat(
+        self,
+        messages: list[dict[str, Any]],
+        tools: list[dict[str, Any]] | None = None,
+        max_tokens: int = 4096,
+    ) -> ChatResponse:
+        contents, system_instruction = self._convert_messages(messages)
+        gemini_tools = self._convert_tools(tools)
+
+        kwargs: dict[str, Any] = {
+            "contents": contents,
+            "generation_config": {"max_output_tokens": max_tokens},
+        }
+        if gemini_tools:
+            kwargs["tools"] = gemini_tools
+        if system_instruction:
+            kwargs["system_instruction"] = system_instruction
+
+        response = self._client.generate_content(**kwargs)
+
+        if not response.candidates:
+            raise RuntimeError("Gemini API returned empty candidates")
+
+        content_text = ""
+        tool_calls = []
+        candidate = response.candidates[0]
+
+        for part in candidate.content.parts:
+            if part.function_call and part.function_call.name:
+                args_dict = dict(part.function_call.args) if part.function_call.args else {}
+                tool_calls.append(
+                    ToolCall(
+                        id=f"call_{uuid.uuid4().hex[:24]}",
+                        name=part.function_call.name,
+                        arguments=json.dumps(args_dict),
+                    )
+                )
+            elif part.text:
+                content_text = part.text
+
+        finish_reason = "stop"
+        if tool_calls:
+            finish_reason = "tool_calls"
+        elif hasattr(candidate.finish_reason, "name"):
+            reason_name = candidate.finish_reason.name
+            if reason_name == "STOP":
+                finish_reason = "stop"
+            elif reason_name == "MAX_TOKENS":
+                finish_reason = "length"
+
+        return ChatResponse(
+            message=ChatMessage(
+                role="assistant",
+                content=content_text if content_text else None,
+                tool_calls=tool_calls,
+            ),
+            finish_reason=finish_reason,
+            input_tokens=getattr(response.usage_metadata, "prompt_token_count", 0)
+            if response.usage_metadata
+            else 0,
+            output_tokens=getattr(response.usage_metadata, "candidates_token_count", 0)
+            if response.usage_metadata
+            else 0,
+        )
+
+
+class QwenProvider:
+    """Provider for Alibaba Qwen API (OpenAI-compatible via DashScope).
+
+    Qwen models are accessed through the DashScope international API endpoint
+    which provides an OpenAI-compatible interface.
+    """
+
+    DASHSCOPE_BASE_URL = "https://dashscope-intl.aliyuncs.com/compatible-mode/v1"
+
+    def __init__(
+        self,
+        model: str,
+        api_key: str | None = None,
+    ) -> None:
+        """Initialize Qwen provider.
+
+        Args:
+            model: Qwen model ID (e.g., 'qwen-plus', 'qwen-turbo', 'qwen-max').
+            api_key: API key. If None, uses DASHSCOPE_API_KEY env var.
+        """
+        self._model = model
+        self._api_key = api_key or os.environ.get("DASHSCOPE_API_KEY")
+        if not self._api_key:
+            raise ValueError(
+                "DashScope API key required. Set DASHSCOPE_API_KEY environment variable "
+                "or pass api_key parameter."
+            )
+        import openai
+
+        self._client = openai.OpenAI(
+            api_key=self._api_key,
+            base_url=self.DASHSCOPE_BASE_URL,
+        )
+
+    @property
+    def model(self) -> str:
+        return self._model
+
+    def get_tool_format(self) -> str:
+        return "openai"
+
+    def chat(
+        self,
+        messages: list[dict[str, Any]],
+        tools: list[dict[str, Any]] | None = None,
+        max_tokens: int = 4096,
+    ) -> ChatResponse:
+        kwargs: dict[str, Any] = {
+            "model": self._model,
+            "messages": messages,
+            "max_tokens": max_tokens,
+        }
+        if tools:
+            kwargs["tools"] = tools
+
+        response = self._client.chat.completions.create(**kwargs)
+
+        if not response.choices:
+            raise RuntimeError("Qwen API returned empty response choices")
+
+        choice = response.choices[0]
+        tool_calls = []
+        if choice.message.tool_calls:
+            for tc in choice.message.tool_calls:
+                tool_calls.append(
+                    ToolCall(
+                        id=tc.id,
+                        name=tc.function.name,
+                        arguments=tc.function.arguments,
+                    )
+                )
+
+        return ChatResponse(
+            message=ChatMessage(
+                role="assistant",
+                content=choice.message.content,
+                tool_calls=tool_calls,
+            ),
+            finish_reason=choice.finish_reason,
+            input_tokens=response.usage.prompt_tokens,
+            output_tokens=response.usage.completion_tokens,
+        )
+
+
 PROVIDER_REGISTRY: dict[str, type] = {
     "anthropic": AnthropicProvider,
+    "openai": OpenAIProvider,
+    "gemini": GeminiProvider,
+    "qwen": QwenProvider,
 }
 
 
@@ -183,7 +496,7 @@ def create_provider(
     """Factory function to create a model provider.
 
     Args:
-        provider_name: Name of the provider (currently only 'anthropic').
+        provider_name: Name of the provider ('anthropic', 'openai', 'gemini', 'qwen').
         model: Model identifier for the provider.
         api_key: Optional API key.
 

mcpbr/sdk.py

@@ -0,0 +1,264 @@
+"""Public Python SDK for mcpbr.
+
+Provides a programmatic interface for running MCP server benchmarks
+from Python code, without requiring the CLI.
+
+Example usage::
+
+    from mcpbr import MCPBenchmark, list_benchmarks, list_models
+
+    # List available benchmarks
+    for b in list_benchmarks():
+        print(b["name"])
+
+    # Create and run a benchmark
+    bench = MCPBenchmark({
+        "mcp_server": {
+            "command": "npx",
+            "args": ["-y", "@modelcontextprotocol/server-filesystem", "{workdir}"],
+        },
+        "benchmark": "humaneval",
+        "model": "sonnet",
+    })
+
+    is_valid, errors = bench.validate()
+    plan = bench.dry_run()
+
+    # Async execution
+    import asyncio
+    result = asyncio.run(bench.run())
+    print(result.success, result.summary)
+"""
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from . import __version__
+from .benchmarks import BENCHMARK_REGISTRY
+from .config import VALID_PROVIDERS, HarnessConfig, load_config
+from .models import SUPPORTED_MODELS, validate_model
+
+
+@dataclass
+class BenchmarkResult:
+    """Result of a benchmark run.
+
+    Attributes:
+        success: Whether the benchmark completed successfully.
+        summary: Aggregated results (e.g., pass rate, resolved count).
+        tasks: Per-task results as a list of dicts.
+        metadata: Run metadata (benchmark name, model, timestamps, etc.).
+        total_cost: Total API cost in USD.
+        total_tokens: Total tokens consumed.
+        duration_seconds: Wall-clock duration of the run.
+    """
+
+    success: bool
+    summary: dict[str, Any]
+    tasks: list[dict[str, Any]]
+    metadata: dict[str, Any]
+    total_cost: float = 0.0
+    total_tokens: int = 0
+    duration_seconds: float = 0.0
+
+
+class MCPBenchmark:
+    """High-level interface for configuring and running MCP benchmarks.
+
+    Can be initialized from a config dict, a YAML file path (str or Path),
+    or an existing HarnessConfig instance.
+
+    Args:
+        config: A dict of config values, a path to a YAML config file
+            (str or Path), or a HarnessConfig instance.
+
+    Raises:
+        FileNotFoundError: If a file path is given and the file does not exist.
+        ValueError: If the config dict is invalid.
+    """
+
+    def __init__(self, config: dict[str, Any] | str | Path | HarnessConfig) -> None:
+        if isinstance(config, HarnessConfig):
+            self.config: HarnessConfig = config
+        elif isinstance(config, (str, Path)):
+            path = Path(config)
+            if not path.exists():
+                raise FileNotFoundError(f"Config file not found: {path}")
+            self.config = load_config(path, warn_security=False)
+        elif isinstance(config, dict):
+            self.config = HarnessConfig(**config)
+        else:
+            raise TypeError(
+                f"config must be a dict, str, Path, or HarnessConfig, got {type(config).__name__}"
+            )
+
+    def validate(self) -> tuple[bool, list[str]]:
+        """Validate the current configuration.
+
+        Checks that the configuration is internally consistent, the model
+        is recognized, and required fields are present.
+
+        Returns:
+            A tuple of (is_valid, list_of_warnings_or_errors).
+        """
+        errors: list[str] = []
+
+        # Validate model is in the supported registry
+        model_valid, model_error = validate_model(self.config.model)
+        if not model_valid:
+            errors.append(f"Model warning: {model_error}")
+
+        # Validate benchmark is in the registry
+        if self.config.benchmark not in BENCHMARK_REGISTRY:
+            errors.append(
+                f"Unknown benchmark: {self.config.benchmark}. "
+                f"Available: {', '.join(BENCHMARK_REGISTRY.keys())}"
+            )
+
+        # Validate provider
+        if self.config.provider not in VALID_PROVIDERS:
+            errors.append(
+                f"Unknown provider: {self.config.provider}. Available: {', '.join(VALID_PROVIDERS)}"
+            )
+
+        is_valid = len(errors) == 0
+        return is_valid, errors
+
+    def dry_run(self) -> dict[str, Any]:
+        """Generate an execution plan without running anything.
+
+        Returns:
+            A dict describing what would be executed, including benchmark,
+            model, provider, MCP server config, and runtime settings.
+        """
+        plan: dict[str, Any] = {
+            "benchmark": self.config.benchmark,
+            "model": self.config.model,
+            "provider": self.config.provider,
+            "agent_harness": self.config.agent_harness,
+            "timeout_seconds": self.config.timeout_seconds,
+            "max_concurrent": self.config.max_concurrent,
+            "max_iterations": self.config.max_iterations,
+            "sample_size": self.config.sample_size,
+        }
+
+        # Include MCP server info
+        if self.config.mcp_server:
+            plan["mcp_server"] = {
+                "command": self.config.mcp_server.command,
+                "args": self.config.mcp_server.args,
+                "name": self.config.mcp_server.name,
+            }
+
+        # Include comparison mode info if applicable
+        if self.config.comparison_mode:
+            plan["comparison_mode"] = True
+            if self.config.mcp_server_a:
+                plan["mcp_server_a"] = {
+                    "command": self.config.mcp_server_a.command,
+                    "args": self.config.mcp_server_a.args,
+                    "name": self.config.mcp_server_a.name,
+                }
+            if self.config.mcp_server_b:
+                plan["mcp_server_b"] = {
+                    "command": self.config.mcp_server_b.command,
+                    "args": self.config.mcp_server_b.args,
+                    "name": self.config.mcp_server_b.name,
+                }
+
+        # Optional settings
+        if self.config.budget is not None:
+            plan["budget"] = self.config.budget
+        if self.config.thinking_budget is not None:
+            plan["thinking_budget"] = self.config.thinking_budget
+        if self.config.agent_prompt is not None:
+            plan["agent_prompt"] = self.config.agent_prompt
+
+        return plan
+
+    async def run(self, **kwargs: Any) -> BenchmarkResult:
+        """Execute the benchmark.
+
+        This is the main entry point for running a benchmark programmatically.
+        It delegates to the internal _execute method, which can be overridden
+        or mocked for testing.
+
+        Args:
+            **kwargs: Additional keyword arguments passed to the executor.
+
+        Returns:
+            BenchmarkResult with the evaluation results.
+        """
+        return await self._execute(**kwargs)
+
+    async def _execute(self, **kwargs: Any) -> BenchmarkResult:
+        """Internal execution method.
+
+        Override or mock this method for testing. In production, this
+        would orchestrate the full benchmark pipeline (task loading,
+        environment creation, agent execution, evaluation).
+
+        Args:
+            **kwargs: Additional keyword arguments.
+
+        Returns:
+            BenchmarkResult with the evaluation results.
+
+        Raises:
+            NotImplementedError: Full execution pipeline is not yet
+                wired into the SDK. Use the CLI for actual runs.
+        """
+        raise NotImplementedError(
+            "Full benchmark execution via the SDK is not yet implemented. "
+            "Use the `mcpbr` CLI for actual benchmark runs, or mock "
+            "MCPBenchmark._execute for testing."
+        )
+
+
+def list_benchmarks() -> list[dict[str, str]]:
+    """List all available benchmarks.
+
+    Returns:
+        A list of dicts, each containing 'name' (the benchmark identifier)
+        and 'class' (the benchmark class name).
+    """
+    return [{"name": name, "class": cls.__name__} for name, cls in BENCHMARK_REGISTRY.items()]
+
+
+def list_providers() -> list[str]:
+    """List all supported model providers.
+
+    Returns:
+        A list of provider name strings.
+    """
+    return list(VALID_PROVIDERS)
+
+
+def list_models() -> list[dict[str, str]]:
+    """List all supported models with their metadata.
+
+    Returns:
+        A list of dicts, each containing 'id', 'provider',
+        'display_name', 'context_window', 'supports_tools', and 'notes'.
+    """
+    return [
+        {
+            "id": info.id,
+            "provider": info.provider,
+            "display_name": info.display_name,
+            "context_window": info.context_window,
+            "supports_tools": info.supports_tools,
+            "notes": info.notes,
+        }
+        for info in SUPPORTED_MODELS.values()
+    ]
+
+
+def get_version() -> str:
+    """Get the current mcpbr version.
+
+    Returns:
+        The version string (e.g., '0.6.0').
+    """
+    return __version__

mcpbr/smoke_test.py

@@ -7,12 +7,13 @@ from dataclasses import dataclass
 from pathlib import Path
 from typing import Any
 
-import docker
 from anthropic import Anthropic
 from rich.console import Console
 from rich.panel import Panel
 from rich.table import Table
 
+import docker
+
 from .config import load_config
 from .config_validator import validate_config
 

{mcpbr-0.5.0.dist-info → mcpbr-0.6.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mcpbr
-Version: 0.5.0
+Version: 0.6.0
 Summary: Model Context Protocol Benchmark Runner - evaluate MCP servers against software engineering benchmarks
 Project-URL: Homepage, https://github.com/greynewell/mcpbr
 Project-URL: Repository, https://github.com/greynewell/mcpbr
@@ -30,6 +30,9 @@ Requires-Dist: pydantic>=2.0.0
 Requires-Dist: pyyaml>=6.0.0
 Requires-Dist: requests>=2.31.0
 Requires-Dist: rich>=13.0.0
+Provides-Extra: all-providers
+Requires-Dist: google-generativeai>=0.3.0; extra == 'all-providers'
+Requires-Dist: openai>=1.0.0; extra == 'all-providers'
 Provides-Extra: dev
 Requires-Dist: pre-commit>=3.0.0; extra == 'dev'
 Requires-Dist: pytest-asyncio>=0.21.0; extra == 'dev'
@@ -40,6 +43,10 @@ Requires-Dist: mkdocs-material>=9.5.0; extra == 'docs'
 Requires-Dist: mkdocs-minify-plugin>=0.7.0; extra == 'docs'
 Requires-Dist: mkdocs>=1.5.0; extra == 'docs'
 Requires-Dist: mkdocstrings[python]>=0.24.0; extra == 'docs'
+Provides-Extra: gemini
+Requires-Dist: google-generativeai>=0.3.0; extra == 'gemini'
+Provides-Extra: openai
+Requires-Dist: openai>=1.0.0; extra == 'openai'
 Description-Content-Type: text/markdown
 
 # mcpbr