mcpbr 0.4.16__py3-none-any.whl → 0.6.0__py3-none-any.whl

mcpbr/sdk.py

@@ -0,0 +1,264 @@
+"""Public Python SDK for mcpbr.
+
+Provides a programmatic interface for running MCP server benchmarks
+from Python code, without requiring the CLI.
+
+Example usage::
+
+    from mcpbr import MCPBenchmark, list_benchmarks, list_models
+
+    # List available benchmarks
+    for b in list_benchmarks():
+        print(b["name"])
+
+    # Create and run a benchmark
+    bench = MCPBenchmark({
+        "mcp_server": {
+            "command": "npx",
+            "args": ["-y", "@modelcontextprotocol/server-filesystem", "{workdir}"],
+        },
+        "benchmark": "humaneval",
+        "model": "sonnet",
+    })
+
+    is_valid, errors = bench.validate()
+    plan = bench.dry_run()
+
+    # Async execution
+    import asyncio
+    result = asyncio.run(bench.run())
+    print(result.success, result.summary)
+"""
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from . import __version__
+from .benchmarks import BENCHMARK_REGISTRY
+from .config import VALID_PROVIDERS, HarnessConfig, load_config
+from .models import SUPPORTED_MODELS, validate_model
+
+
+@dataclass
+class BenchmarkResult:
+    """Result of a benchmark run.
+
+    Attributes:
+        success: Whether the benchmark completed successfully.
+        summary: Aggregated results (e.g., pass rate, resolved count).
+        tasks: Per-task results as a list of dicts.
+        metadata: Run metadata (benchmark name, model, timestamps, etc.).
+        total_cost: Total API cost in USD.
+        total_tokens: Total tokens consumed.
+        duration_seconds: Wall-clock duration of the run.
+    """
+
+    success: bool
+    summary: dict[str, Any]
+    tasks: list[dict[str, Any]]
+    metadata: dict[str, Any]
+    total_cost: float = 0.0
+    total_tokens: int = 0
+    duration_seconds: float = 0.0
+
+
+class MCPBenchmark:
+    """High-level interface for configuring and running MCP benchmarks.
+
+    Can be initialized from a config dict, a YAML file path (str or Path),
+    or an existing HarnessConfig instance.
+
+    Args:
+        config: A dict of config values, a path to a YAML config file
+            (str or Path), or a HarnessConfig instance.
+
+    Raises:
+        FileNotFoundError: If a file path is given and the file does not exist.
+        ValueError: If the config dict is invalid.
+    """
+
+    def __init__(self, config: dict[str, Any] | str | Path | HarnessConfig) -> None:
+        if isinstance(config, HarnessConfig):
+            self.config: HarnessConfig = config
+        elif isinstance(config, (str, Path)):
+            path = Path(config)
+            if not path.exists():
+                raise FileNotFoundError(f"Config file not found: {path}")
+            self.config = load_config(path, warn_security=False)
+        elif isinstance(config, dict):
+            self.config = HarnessConfig(**config)
+        else:
+            raise TypeError(
+                f"config must be a dict, str, Path, or HarnessConfig, got {type(config).__name__}"
+            )
+
+    def validate(self) -> tuple[bool, list[str]]:
+        """Validate the current configuration.
+
+        Checks that the configuration is internally consistent, the model
+        is recognized, and required fields are present.
+
+        Returns:
+            A tuple of (is_valid, list_of_warnings_or_errors).
+        """
+        errors: list[str] = []
+
+        # Validate model is in the supported registry
+        model_valid, model_error = validate_model(self.config.model)
+        if not model_valid:
+            errors.append(f"Model warning: {model_error}")
+
+        # Validate benchmark is in the registry
+        if self.config.benchmark not in BENCHMARK_REGISTRY:
+            errors.append(
+                f"Unknown benchmark: {self.config.benchmark}. "
+                f"Available: {', '.join(BENCHMARK_REGISTRY.keys())}"
+            )
+
+        # Validate provider
+        if self.config.provider not in VALID_PROVIDERS:
+            errors.append(
+                f"Unknown provider: {self.config.provider}. Available: {', '.join(VALID_PROVIDERS)}"
+            )
+
+        is_valid = len(errors) == 0
+        return is_valid, errors
+
+    def dry_run(self) -> dict[str, Any]:
+        """Generate an execution plan without running anything.
+
+        Returns:
+            A dict describing what would be executed, including benchmark,
+            model, provider, MCP server config, and runtime settings.
+        """
+        plan: dict[str, Any] = {
+            "benchmark": self.config.benchmark,
+            "model": self.config.model,
+            "provider": self.config.provider,
+            "agent_harness": self.config.agent_harness,
+            "timeout_seconds": self.config.timeout_seconds,
+            "max_concurrent": self.config.max_concurrent,
+            "max_iterations": self.config.max_iterations,
+            "sample_size": self.config.sample_size,
+        }
+
+        # Include MCP server info
+        if self.config.mcp_server:
+            plan["mcp_server"] = {
+                "command": self.config.mcp_server.command,
+                "args": self.config.mcp_server.args,
+                "name": self.config.mcp_server.name,
+            }
+
+        # Include comparison mode info if applicable
+        if self.config.comparison_mode:
+            plan["comparison_mode"] = True
+            if self.config.mcp_server_a:
+                plan["mcp_server_a"] = {
+                    "command": self.config.mcp_server_a.command,
+                    "args": self.config.mcp_server_a.args,
+                    "name": self.config.mcp_server_a.name,
+                }
+            if self.config.mcp_server_b:
+                plan["mcp_server_b"] = {
+                    "command": self.config.mcp_server_b.command,
+                    "args": self.config.mcp_server_b.args,
+                    "name": self.config.mcp_server_b.name,
+                }
+
+        # Optional settings
+        if self.config.budget is not None:
+            plan["budget"] = self.config.budget
+        if self.config.thinking_budget is not None:
+            plan["thinking_budget"] = self.config.thinking_budget
+        if self.config.agent_prompt is not None:
+            plan["agent_prompt"] = self.config.agent_prompt
+
+        return plan
+
+    async def run(self, **kwargs: Any) -> BenchmarkResult:
+        """Execute the benchmark.
+
+        This is the main entry point for running a benchmark programmatically.
+        It delegates to the internal _execute method, which can be overridden
+        or mocked for testing.
+
+        Args:
+            **kwargs: Additional keyword arguments passed to the executor.
+
+        Returns:
+            BenchmarkResult with the evaluation results.
+        """
+        return await self._execute(**kwargs)
+
+    async def _execute(self, **kwargs: Any) -> BenchmarkResult:
+        """Internal execution method.
+
+        Override or mock this method for testing. In production, this
+        would orchestrate the full benchmark pipeline (task loading,
+        environment creation, agent execution, evaluation).
+
+        Args:
+            **kwargs: Additional keyword arguments.
+
+        Returns:
+            BenchmarkResult with the evaluation results.
+
+        Raises:
+            NotImplementedError: Full execution pipeline is not yet
+                wired into the SDK. Use the CLI for actual runs.
+        """
+        raise NotImplementedError(
+            "Full benchmark execution via the SDK is not yet implemented. "
+            "Use the `mcpbr` CLI for actual benchmark runs, or mock "
+            "MCPBenchmark._execute for testing."
+        )
+
+
+def list_benchmarks() -> list[dict[str, str]]:
+    """List all available benchmarks.
+
+    Returns:
+        A list of dicts, each containing 'name' (the benchmark identifier)
+        and 'class' (the benchmark class name).
+    """
+    return [{"name": name, "class": cls.__name__} for name, cls in BENCHMARK_REGISTRY.items()]
+
+
+def list_providers() -> list[str]:
+    """List all supported model providers.
+
+    Returns:
+        A list of provider name strings.
+    """
+    return list(VALID_PROVIDERS)
+
+
+def list_models() -> list[dict[str, str]]:
+    """List all supported models with their metadata.
+
+    Returns:
+        A list of dicts, each containing 'id', 'provider',
+        'display_name', 'context_window', 'supports_tools', and 'notes'.
+    """
+    return [
+        {
+            "id": info.id,
+            "provider": info.provider,
+            "display_name": info.display_name,
+            "context_window": info.context_window,
+            "supports_tools": info.supports_tools,
+            "notes": info.notes,
+        }
+        for info in SUPPORTED_MODELS.values()
+    ]
+
+
+def get_version() -> str:
+    """Get the current mcpbr version.
+
+    Returns:
+        The version string (e.g., '0.6.0').
+    """
+    return __version__

mcpbr/smoke_test.py

@@ -7,12 +7,13 @@ from dataclasses import dataclass
 from pathlib import Path
 from typing import Any
 
-import docker
 from anthropic import Anthropic
 from rich.console import Console
 from rich.panel import Panel
 from rich.table import Table
 
+import docker
+
 from .config import load_config
 from .config_validator import validate_config
 

mcpbr/task_batching.py

@@ -0,0 +1,403 @@
+"""Task batching with smart scheduling for efficient batch execution.
+
+Groups similar benchmark tasks to minimize Docker container restarts and
+maximize resource reuse. Supports multiple batching strategies including
+repo-based, image-based, category-based, fixed-size, and adaptive grouping.
+"""
+
+import uuid
+from collections import defaultdict
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any
+
+# Estimated overhead per Docker container restart in seconds
+_CONTAINER_RESTART_OVERHEAD_SECONDS = 30.0
+
+
+class BatchStrategy(Enum):
+    """Strategy for grouping tasks into batches.
+
+    Attributes:
+        BY_REPO: Group tasks that share the same repository.
+        BY_IMAGE: Group tasks that require the same Docker image.
+        BY_CATEGORY: Group tasks that belong to the same benchmark category.
+        FIXED_SIZE: Split tasks into fixed-size chunks regardless of similarity.
+        ADAPTIVE: Dynamically size batches based on task similarity signals.
+    """
+
+    BY_REPO = "by_repo"
+    BY_IMAGE = "by_image"
+    BY_CATEGORY = "by_category"
+    FIXED_SIZE = "fixed_size"
+    ADAPTIVE = "adaptive"
+
+
+@dataclass
+class TaskBatch:
+    """A batch of grouped tasks for efficient execution.
+
+    Attributes:
+        batch_id: Unique identifier for this batch.
+        tasks: List of task dictionaries in this batch.
+        common_image: Shared Docker image if all tasks use the same one, else None.
+        common_repo: Shared repository if all tasks target the same repo, else None.
+        batch_size: Number of tasks in this batch.
+        estimated_savings_seconds: Estimated time saved by batching vs individual execution.
+    """
+
+    batch_id: str
+    tasks: list[dict[str, Any]]
+    common_image: str | None = None
+    common_repo: str | None = None
+    batch_size: int = 0
+    estimated_savings_seconds: float = 0.0
+
+    def __post_init__(self) -> None:
+        """Compute batch_size from tasks if not explicitly set."""
+        if self.batch_size == 0 and self.tasks:
+            self.batch_size = len(self.tasks)
+
+
+@dataclass
+class BatchSavings:
+    """Estimated savings from batching tasks.
+
+    Attributes:
+        total_batches: Total number of batches created.
+        avg_batch_size: Average number of tasks per batch.
+        estimated_container_reuse: Number of container restarts avoided.
+        estimated_time_saved_seconds: Total estimated time saved in seconds.
+    """
+
+    total_batches: int = 0
+    avg_batch_size: float = 0.0
+    estimated_container_reuse: int = 0
+    estimated_time_saved_seconds: float = 0.0
+
+
+class TaskBatcher:
+    """Groups benchmark tasks into batches for efficient execution.
+
+    Batching reduces Docker container restarts by grouping tasks that share
+    common requirements (repository, image, category). Supports multiple
+    strategies and configurable batch sizes.
+
+    Args:
+        strategy: Batching strategy to use.
+        max_batch_size: Maximum number of tasks per batch.
+        min_batch_size: Minimum number of tasks to form a batch. Groups smaller
+            than this are still returned as batches (no tasks are dropped).
+
+    Example:
+        >>> batcher = TaskBatcher(strategy=BatchStrategy.BY_REPO, max_batch_size=5)
+        >>> tasks = [{"instance_id": "t1", "repo": "org/repo1"}, ...]
+        >>> batches = batcher.batch(tasks)
+        >>> print(batcher.preview(batches))
+    """
+
+    def __init__(
+        self,
+        strategy: BatchStrategy = BatchStrategy.BY_REPO,
+        max_batch_size: int = 10,
+        min_batch_size: int = 2,
+    ) -> None:
+        """Initialize the TaskBatcher.
+
+        Args:
+            strategy: Batching strategy to use.
+            max_batch_size: Maximum number of tasks per batch.
+            min_batch_size: Minimum number of tasks to form a batch.
+
+        Raises:
+            ValueError: If max_batch_size < 1 or min_batch_size < 1 or
+                min_batch_size > max_batch_size.
+        """
+        if max_batch_size < 1:
+            raise ValueError(f"max_batch_size must be >= 1, got {max_batch_size}")
+        if min_batch_size < 1:
+            raise ValueError(f"min_batch_size must be >= 1, got {min_batch_size}")
+        if min_batch_size > max_batch_size:
+            raise ValueError(
+                f"min_batch_size ({min_batch_size}) must be <= max_batch_size ({max_batch_size})"
+            )
+        self.strategy = strategy
+        self.max_batch_size = max_batch_size
+        self.min_batch_size = min_batch_size
+
+    def batch(self, tasks: list[dict[str, Any]]) -> list[TaskBatch]:
+        """Group tasks into batches using the configured strategy.
+
+        Args:
+            tasks: List of task dictionaries to batch. Each task should have
+                at minimum an ``instance_id`` key. Depending on the strategy,
+                ``repo``, ``image``, and ``category`` fields are also used.
+
+        Returns:
+            List of TaskBatch objects. Every input task appears in exactly one
+            batch. Batches are sorted by descending size for scheduling efficiency.
+        """
+        if not tasks:
+            return []
+
+        if self.strategy == BatchStrategy.BY_REPO:
+            return self._batch_by_field(tasks, "repo")
+        elif self.strategy == BatchStrategy.BY_IMAGE:
+            return self._batch_by_field(tasks, "image")
+        elif self.strategy == BatchStrategy.BY_CATEGORY:
+            return self._batch_by_field(tasks, "category")
+        elif self.strategy == BatchStrategy.FIXED_SIZE:
+            return self._batch_fixed_size(tasks)
+        elif self.strategy == BatchStrategy.ADAPTIVE:
+            return self._batch_adaptive(tasks)
+        else:
+            raise ValueError(f"Unknown batch strategy: {self.strategy}")
+
+    def estimate_savings(self, batches: list[TaskBatch]) -> BatchSavings:
+        """Estimate time saved by batching compared to individual execution.
+
+        The savings come primarily from container reuse: tasks in the same batch
+        can share a Docker container instead of each requiring a fresh one.
+
+        Args:
+            batches: List of TaskBatch objects to analyze.
+
+        Returns:
+            BatchSavings with estimated metrics.
+        """
+        if not batches:
+            return BatchSavings()
+
+        total_tasks = sum(b.batch_size for b in batches)
+        total_batches = len(batches)
+        avg_batch_size = total_tasks / total_batches if total_batches > 0 else 0.0
+
+        # Without batching, each task needs its own container restart.
+        # With batching, only the first task in each batch needs a restart.
+        container_reuse = total_tasks - total_batches
+        time_saved = container_reuse * _CONTAINER_RESTART_OVERHEAD_SECONDS
+
+        return BatchSavings(
+            total_batches=total_batches,
+            avg_batch_size=round(avg_batch_size, 2),
+            estimated_container_reuse=container_reuse,
+            estimated_time_saved_seconds=round(time_saved, 2),
+        )
+
+    def preview(self, batches: list[TaskBatch]) -> str:
+        """Generate a formatted preview of the batching plan.
+
+        Args:
+            batches: List of TaskBatch objects to preview.
+
+        Returns:
+            Human-readable string summarizing the batches and estimated savings.
+        """
+        if not batches:
+            return "No batches to preview."
+
+        savings = self.estimate_savings(batches)
+        lines: list[str] = []
+        lines.append(f"Batch Plan ({self.strategy.value})")
+        lines.append("=" * 50)
+        lines.append(f"Total batches: {savings.total_batches}")
+        lines.append(f"Average batch size: {savings.avg_batch_size}")
+        lines.append(f"Estimated container reuse: {savings.estimated_container_reuse}")
+        lines.append(f"Estimated time saved: {savings.estimated_time_saved_seconds:.1f}s")
+        lines.append("")
+
+        for i, b in enumerate(batches, 1):
+            label_parts: list[str] = []
+            if b.common_repo:
+                label_parts.append(f"repo={b.common_repo}")
+            if b.common_image:
+                label_parts.append(f"image={b.common_image}")
+            label = ", ".join(label_parts) if label_parts else "mixed"
+
+            lines.append(f"  Batch {i}: {b.batch_size} tasks ({label})")
+            task_ids = [t.get("instance_id", "?") for t in b.tasks[:5]]
+            if b.batch_size > 5:
+                task_ids.append(f"... +{b.batch_size - 5} more")
+            for tid in task_ids:
+                lines.append(f"    - {tid}")
+
+        return "\n".join(lines)
+
+    # ------------------------------------------------------------------
+    # Private helpers
+    # ------------------------------------------------------------------
+
+    def _batch_by_field(self, tasks: list[dict[str, Any]], field_name: str) -> list[TaskBatch]:
+        """Group tasks by a shared field, then split into max-sized chunks.
+
+        Args:
+            tasks: List of task dictionaries.
+            field_name: Key to group tasks by (e.g. "repo", "image", "category").
+
+        Returns:
+            Sorted list of TaskBatch objects.
+        """
+        groups: dict[str, list[dict[str, Any]]] = defaultdict(list)
+        for task in tasks:
+            key = str(task.get(field_name, "_ungrouped_"))
+            groups[key].append(task)
+
+        batches: list[TaskBatch] = []
+        for key, group_tasks in sorted(groups.items()):
+            for chunk in self._split_into_chunks(group_tasks):
+                common_image = self._common_value(chunk, "image")
+                common_repo = self._common_value(chunk, "repo")
+                savings = self._estimate_batch_savings(len(chunk))
+                batches.append(
+                    TaskBatch(
+                        batch_id=str(uuid.uuid4()),
+                        tasks=chunk,
+                        common_image=common_image,
+                        common_repo=common_repo,
+                        batch_size=len(chunk),
+                        estimated_savings_seconds=savings,
+                    )
+                )
+
+        # Sort largest first for better scheduling
+        batches.sort(key=lambda b: b.batch_size, reverse=True)
+        return batches
+
+    def _batch_fixed_size(self, tasks: list[dict[str, Any]]) -> list[TaskBatch]:
+        """Split tasks into fixed-size chunks.
+
+        Args:
+            tasks: List of task dictionaries.
+
+        Returns:
+            Sorted list of TaskBatch objects.
+        """
+        batches: list[TaskBatch] = []
+        for chunk in self._split_into_chunks(tasks):
+            common_image = self._common_value(chunk, "image")
+            common_repo = self._common_value(chunk, "repo")
+            savings = self._estimate_batch_savings(len(chunk))
+            batches.append(
+                TaskBatch(
+                    batch_id=str(uuid.uuid4()),
+                    tasks=chunk,
+                    common_image=common_image,
+                    common_repo=common_repo,
+                    batch_size=len(chunk),
+                    estimated_savings_seconds=savings,
+                )
+            )
+        return batches
+
+    def _batch_adaptive(self, tasks: list[dict[str, Any]]) -> list[TaskBatch]:
+        """Adaptively group tasks based on multi-field similarity.
+
+        Tasks are first grouped by a composite key of all available grouping
+        fields (repo, image, category). Groups that share more fields get
+        larger batch sizes (up to max_batch_size). Groups with no shared
+        fields get smaller batches (down toward min_batch_size).
+
+        Args:
+            tasks: List of task dictionaries.
+
+        Returns:
+            Sorted list of TaskBatch objects.
+        """
+        # Build composite similarity groups
+        similarity_fields = ["repo", "image", "category"]
+        groups: dict[str, list[dict[str, Any]]] = defaultdict(list)
+        for task in tasks:
+            parts = []
+            for f in similarity_fields:
+                parts.append(str(task.get(f, "_")))
+            key = "|".join(parts)
+            groups[key].append(task)
+
+        batches: list[TaskBatch] = []
+        for key, group_tasks in sorted(groups.items()):
+            # Determine how many fields are shared (non-default)
+            key_parts = key.split("|")
+            shared_count = sum(1 for p in key_parts if p != "_")
+
+            # Scale batch size based on similarity: more shared fields -> larger batches
+            similarity_ratio = shared_count / len(similarity_fields) if similarity_fields else 0
+            adaptive_max = self.min_batch_size + int(
+                (self.max_batch_size - self.min_batch_size) * similarity_ratio
+            )
+            adaptive_max = max(adaptive_max, self.min_batch_size)
+
+            for chunk in self._split_into_chunks(group_tasks, max_size=adaptive_max):
+                common_image = self._common_value(chunk, "image")
+                common_repo = self._common_value(chunk, "repo")
+                savings = self._estimate_batch_savings(len(chunk))
+                batches.append(
+                    TaskBatch(
+                        batch_id=str(uuid.uuid4()),
+                        tasks=chunk,
+                        common_image=common_image,
+                        common_repo=common_repo,
+                        batch_size=len(chunk),
+                        estimated_savings_seconds=savings,
+                    )
+                )
+
+        batches.sort(key=lambda b: b.batch_size, reverse=True)
+        return batches
+
+    def _split_into_chunks(
+        self,
+        tasks: list[dict[str, Any]],
+        max_size: int | None = None,
+    ) -> list[list[dict[str, Any]]]:
+        """Split a list of tasks into chunks of at most max_size.
+
+        Args:
+            tasks: Tasks to split.
+            max_size: Override for maximum chunk size. Defaults to self.max_batch_size.
+
+        Returns:
+            List of task sublists.
+        """
+        size = max_size if max_size is not None else self.max_batch_size
+        if size < 1:
+            size = 1
+        chunks: list[list[dict[str, Any]]] = []
+        for i in range(0, len(tasks), size):
+            chunks.append(tasks[i : i + size])
+        return chunks
+
+    @staticmethod
+    def _common_value(tasks: list[dict[str, Any]], field_name: str) -> str | None:
+        """Return the shared value for a field if all tasks agree, else None.
+
+        Args:
+            tasks: List of task dictionaries.
+            field_name: Key to check.
+
+        Returns:
+            The common value string, or None if tasks differ or field is absent.
+        """
+        if not tasks:
+            return None
+        values = {t.get(field_name) for t in tasks}
+        values.discard(None)
+        if len(values) == 1:
+            return str(values.pop())
+        return None
+
+    @staticmethod
+    def _estimate_batch_savings(batch_size: int) -> float:
+        """Estimate time saved for a single batch.
+
+        Each additional task in a batch beyond the first avoids one container
+        restart.
+
+        Args:
+            batch_size: Number of tasks in the batch.
+
+        Returns:
+            Estimated time saved in seconds.
+        """
+        if batch_size <= 1:
+            return 0.0
+        return (batch_size - 1) * _CONTAINER_RESTART_OVERHEAD_SECONDS