mcpbr 0.4.16__tar.gz → 0.6.0__tar.gz

{mcpbr-0.4.16 → mcpbr-0.6.0}/.claude-plugin/marketplace.json

@@ -1,7 +1,7 @@
 {
   "$schema": "https://anthropic.com/claude-code/marketplace.schema.json",
   "name": "mcpbr",
-  "version": "0.4.16",
+  "version": "0.6.0",
   "description": "mcpbr - MCP Benchmark Runner plugin marketplace",
   "owner": {
     "name": "mcpbr Contributors",
@@ -11,7 +11,7 @@
     {
       "name": "mcpbr",
       "description": "Expert benchmark runner for MCP servers using mcpbr. Handles Docker checks, config generation, and result parsing.",
-      "version": "0.4.16",
+      "version": "0.6.0",
       "author": {
         "name": "mcpbr Contributors"
       },

{mcpbr-0.4.16 → mcpbr-0.6.0}/.claude-plugin/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@greynewell/mcpbr-claude-plugin",
-  "version": "0.4.16",
+  "version": "0.6.0",
   "description": "Claude Code plugin for mcpbr - Expert benchmark runner for MCP servers with specialized skills",
   "keywords": [
     "claude-code",

{mcpbr-0.4.16 → mcpbr-0.6.0}/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "mcpbr",
-  "version": "0.4.16",
+  "version": "0.6.0",
   "description": "Expert benchmark runner for MCP servers using mcpbr. Handles Docker checks, config generation, and result parsing.",
   "schema_version": "1.0"
 }

mcpbr-0.6.0/.dockerignore

@@ -0,0 +1,55 @@
+# Git
+.git
+.gitignore
+
+# Python
+__pycache__
+*.pyc
+*.pyo
+*.egg-info
+dist/
+build/
+.eggs/
+*.egg
+
+# Virtual environments
+.venv
+venv
+env
+
+# IDE
+.vscode
+.idea
+*.swp
+*.swo
+
+# Testing
+.pytest_cache
+.coverage
+htmlcov/
+.tox
+
+# Documentation
+docs/
+site/
+mkdocs.yml
+
+# CI/CD
+.github/
+ci-templates/
+
+# Docker (avoid recursive copies)
+docker/results/
+
+# Node
+node_modules/
+npm-debug.log
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Secrets
+.env
+.env.*
+credentials.json

mcpbr-0.6.0/.gitattributes

@@ -0,0 +1,2 @@
+# Ensure Windows batch files use CRLF line endings
+conda/bld.bat text eol=crlf

mcpbr-0.6.0/.github/workflows/build-binaries.yml

@@ -0,0 +1,83 @@
+name: Build Binaries
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Release tag to build (e.g., v0.6.0)"
+        required: false
+        type: string
+
+permissions:
+  contents: write
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-latest
+            platform: linux
+            arch: x86_64
+            artifact-name: mcpbr-linux-x86_64
+          - os: macos-latest
+            platform: macos
+            arch: arm64
+            artifact-name: mcpbr-macos-arm64
+          - os: macos-13-large
+            platform: macos
+            arch: x86_64
+            artifact-name: mcpbr-macos-x86_64
+          - os: windows-latest
+            platform: windows
+            arch: x86_64
+            artifact-name: mcpbr-windows-x86_64
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.inputs.tag || github.ref }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install pyinstaller
+          pip install -e "."
+
+      - name: Build binary (Unix)
+        if: matrix.platform != 'windows'
+        run: |
+          pyinstaller mcpbr.spec
+          cd dist
+          tar -czf ${{ matrix.artifact-name }}.tar.gz mcpbr
+          cd ..
+
+      - name: Build binary (Windows)
+        if: matrix.platform == 'windows'
+        run: |
+          pyinstaller mcpbr.spec
+          cd dist
+          Compress-Archive -Path mcpbr.exe -DestinationPath ${{ matrix.artifact-name }}.zip
+          cd ..
+        shell: pwsh
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.artifact-name }}
+          path: dist/mcpbr-*.*
+
+      - name: Upload to release
+        if: github.event_name == 'release'
+        uses: softprops/action-gh-release@v2
+        with:
+          files: dist/mcpbr-*.*

{mcpbr-0.4.16 → mcpbr-0.6.0}/.gitignore

@@ -27,6 +27,7 @@ wheels/
 # PyInstaller
 *.manifest
 *.spec
+!mcpbr.spec
 
 # Installer logs
 pip-log.txt

{mcpbr-0.4.16 → mcpbr-0.6.0}/.pre-commit-config.yaml

@@ -22,5 +22,6 @@ repos:
       - id: trailing-whitespace
       - id: end-of-file-fixer
       - id: check-yaml
+        exclude: ^conda/meta\.yaml$
       - id: check-added-large-files
       - id: check-merge-conflict

{mcpbr-0.4.16 → mcpbr-0.6.0}/CHANGELOG.md

@@ -7,6 +7,59 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.6.0] - 2026-02-05
+
+### Added
+
+- **Graceful degradation** (#70): Fault-tolerant task execution with failure isolation, classification (transient/permanent/unknown), configurable `continue_on_error` and `max_failures` policies, execution checkpointing for crash recovery, and partial report generation
+  - New config fields: `continue_on_error`, `max_failures`, `checkpoint_interval`, `resume_from_checkpoint`
+- **Multi-provider support** (#229): Added OpenAI, Google Gemini, and Alibaba Qwen as model providers alongside Anthropic
+  - `OpenAIProvider` for GPT-4o, GPT-4 Turbo, and GPT-4o Mini models
+  - `GeminiProvider` for Gemini 2.0 Flash, Gemini 1.5 Pro, and Gemini 1.5 Flash models
+  - `QwenProvider` for Qwen Plus, Qwen Turbo, and Qwen Max models via DashScope API
+  - New optional dependencies: `openai`, `gemini`, `all-providers` extras
+  - Pricing data for all 9 new models
+  - Model registry entries with context window and tool support metadata
+- **Multi-language support** (#230): Cross-language benchmark execution for Python, JavaScript, TypeScript, Java, and Go
+  - Per-language Docker images, run/compile commands, and test framework configs
+  - Automatic language detection from filenames and code patterns
+  - Cross-language metrics aggregation
+- **Structured logging** (#231): JSON and human-readable log formatters with contextual metadata
+  - File rotation, configurable log levels via `MCPBR_LOG_LEVEL` env var
+  - `LogContext` context manager for injecting task/benchmark fields into log records
+- **Public Python SDK** (#232): Programmatic API for configuring and running benchmarks
+  - `MCPBenchmark` class with config from dict, YAML, or `HarnessConfig`
+  - `list_benchmarks()`, `list_providers()`, `list_models()`, `get_version()` helpers
+  - Exported in top-level `mcpbr` package for `from mcpbr import MCPBenchmark`
+- **Platform distribution files**: Docker, Conda, Homebrew, GitHub Actions, and CI templates
+  - `Dockerfile.app` multi-stage build for container deployment
+  - `docker/docker-compose.yml` for multi-container orchestration
+  - `conda/meta.yaml` recipe for Conda packaging
+  - `action/action.yml` GitHub Action with basic and matrix examples
+  - `ci-templates/` for GitLab CI and CircleCI integration
+
+## [0.5.0] - 2026-02-05
+
+### Added
+
+- **Real-time web dashboard** (#58): Live monitoring of benchmark evaluations via `DashboardServer` with FastAPI + WebSocket, task progress, resolution rate, ETA, and pause/resume/cancel controls
+- **Interactive configuration wizard** (#74): Step-by-step CLI wizard for creating config files with presets (filesystem, web-search, database), model/benchmark selection, and MCP server setup
+- **Dry-run mode** (#84): Preview evaluation plan without executing — shows tasks, estimated cost/time, validates config, checks Docker and MCP server availability
+- **Task prioritization and scheduling** (#92): Intelligent task ordering with speed-first, cost-first, coverage-first, and custom scoring strategies
+- **Color and formatting options** (#105): Configurable output themes (default, minimal, plain) with NO_COLOR convention support and MCPBR_THEME env var
+- **Docker image pre-warming** (#128): Pre-pull Docker images in parallel before evaluation starts with progress reporting and cache detection
+- **Result streaming to external storage** (#131): Stream results as tasks complete to local JSONL files, S3-compatible storage, or webhooks with buffering and retry
+- **Memory-efficient large dataset handling** (#134): Streaming and chunked loading of large HuggingFace datasets with memory monitoring and automatic chunk-size adaptation
+- **Task batching with smart scheduling** (#137): Group similar tasks by repo/image/category to minimize Docker container restarts with adaptive batch sizing
+- **Resource limit configuration** (#139): Configure CPU, memory, disk, PID, and network limits for Docker containers with monitoring and violation reporting
+- **Configuration migration tool** (#195): Detect and migrate old config formats (V1→V4) with dry-run preview, backup, and chained migration steps
+- **Docker image caching optimization** (#228): LRU cache management with size limits, usage tracking, eviction, warmup recommendations, and dangling image cleanup
+
+### Fixed
+
+- **Zero-cost metrics on evaluation timeout** (#374): Agent metrics (cost, tokens, iterations) were discarded when `benchmark.evaluate()` timed out after the agent had already completed successfully. Now preserves agent results when available.
+- **Process hang after evaluation completes** (#374): `asyncio.run()` blocked indefinitely during cleanup because Docker SDK urllib3 background threads kept the default executor alive. Now force-shuts down the executor with `wait=False`.
+
 ## [0.4.16] - 2026-02-05
 
 ### Added
@@ -729,6 +782,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 [0.3.14]: https://github.com/greynewell/mcpbr/releases/tag/v0.3.14
 [0.3.13]: https://github.com/greynewell/mcpbr/releases/tag/v0.3.13
 [0.3.12]: https://github.com/greynewell/mcpbr/releases/tag/v0.3.12
+[0.6.0]: https://github.com/greynewell/mcpbr/releases/tag/v0.6.0
+[0.5.0]: https://github.com/greynewell/mcpbr/releases/tag/v0.5.0
 [0.4.16]: https://github.com/greynewell/mcpbr/releases/tag/v0.4.16
 [0.3.11]: https://github.com/greynewell/mcpbr/releases/tag/v0.3.11
 [0.3.10]: https://github.com/greynewell/mcpbr/releases/tag/v0.3.10

mcpbr-0.6.0/Dockerfile.app

@@ -0,0 +1,61 @@
+# Multi-stage Dockerfile for running mcpbr CLI
+# This is NOT the task environment Dockerfile - see Dockerfile for that.
+# This image packages the mcpbr benchmark runner itself.
+
+# Stage 1: Build
+FROM python:3.11-slim AS builder
+
+WORKDIR /build
+
+# Install build dependencies
+RUN pip install --no-cache-dir build hatchling
+
+# Copy project files
+COPY pyproject.toml README.md LICENSE ./
+COPY src/ src/
+
+# Build the wheel
+RUN python -m build --wheel --outdir /build/dist
+
+# Stage 2: Runtime
+FROM python:3.11-slim
+
+LABEL maintainer="mcpbr Contributors"
+LABEL org.opencontainers.image.source="https://github.com/greynewell/mcpbr"
+LABEL org.opencontainers.image.description="MCP Benchmark Runner - evaluate MCP servers against software engineering benchmarks"
+LABEL org.opencontainers.image.licenses="MIT"
+
+# Install runtime system dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git \
+    curl \
+    ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+# Create non-root user
+RUN groupadd --gid 1000 mcpbr && \
+    useradd --uid 1000 --gid mcpbr --shell /bin/bash --create-home mcpbr
+
+WORKDIR /home/mcpbr
+
+# Install the built wheel from the builder stage
+COPY --from=builder /build/dist/*.whl /tmp/
+RUN pip install --no-cache-dir /tmp/*.whl && \
+    rm -f /tmp/*.whl
+
+# Copy entrypoint
+COPY docker/entrypoint.sh /usr/local/bin/entrypoint.sh
+RUN chmod +x /usr/local/bin/entrypoint.sh
+
+# Create directories for configs and results
+RUN mkdir -p /home/mcpbr/configs /home/mcpbr/results && \
+    chown -R mcpbr:mcpbr /home/mcpbr
+
+# Switch to non-root user
+USER mcpbr
+
+# Mount points for user configs and results
+VOLUME ["/home/mcpbr/configs", "/home/mcpbr/results"]
+
+ENTRYPOINT ["entrypoint.sh"]
+CMD ["--help"]

mcpbr-0.6.0/Formula/mcpbr.rb

@@ -0,0 +1,51 @@
+class Mcpbr < Formula
+  include Language::Python::Virtualenv
+
+  desc "Model Context Protocol Benchmark Runner - evaluate MCP servers against software engineering benchmarks"
+  homepage "https://github.com/greynewell/mcpbr"
+  # NOTE: Update URL and sha256 when publishing a release.
+  # Run: curl -sL <url> | shasum -a 256
+  url "https://files.pythonhosted.org/packages/source/m/mcpbr/mcpbr-0.6.0.tar.gz"
+  sha256 "PLACEHOLDER_SHA256_REPLACE_ON_RELEASE"
+  license "MIT"
+
+  depends_on "python@3.11"
+
+  resource "anthropic" do
+    url "https://files.pythonhosted.org/packages/source/a/anthropic/anthropic-0.40.0.tar.gz"
+    sha256 "PLACEHOLDER_SHA256"
+  end
+
+  resource "click" do
+    url "https://files.pythonhosted.org/packages/source/c/click/click-8.1.7.tar.gz"
+    sha256 "PLACEHOLDER_SHA256"
+  end
+
+  resource "docker" do
+    url "https://files.pythonhosted.org/packages/source/d/docker/docker-7.0.0.tar.gz"
+    sha256 "PLACEHOLDER_SHA256"
+  end
+
+  resource "pydantic" do
+    url "https://files.pythonhosted.org/packages/source/p/pydantic/pydantic-2.0.0.tar.gz"
+    sha256 "PLACEHOLDER_SHA256"
+  end
+
+  resource "pyyaml" do
+    url "https://files.pythonhosted.org/packages/source/p/PyYAML/PyYAML-6.0.1.tar.gz"
+    sha256 "PLACEHOLDER_SHA256"
+  end
+
+  resource "rich" do
+    url "https://files.pythonhosted.org/packages/source/r/rich/rich-13.0.0.tar.gz"
+    sha256 "PLACEHOLDER_SHA256"
+  end
+
+  def install
+    virtualenv_install_with_resources
+  end
+
+  test do
+    assert_match "mcpbr", shell_output("#{bin}/mcpbr --version")
+  end
+end

{mcpbr-0.4.16 → mcpbr-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mcpbr
-Version: 0.4.16
+Version: 0.6.0
 Summary: Model Context Protocol Benchmark Runner - evaluate MCP servers against software engineering benchmarks
 Project-URL: Homepage, https://github.com/greynewell/mcpbr
 Project-URL: Repository, https://github.com/greynewell/mcpbr
@@ -30,6 +30,9 @@ Requires-Dist: pydantic>=2.0.0
 Requires-Dist: pyyaml>=6.0.0
 Requires-Dist: requests>=2.31.0
 Requires-Dist: rich>=13.0.0
+Provides-Extra: all-providers
+Requires-Dist: google-generativeai>=0.3.0; extra == 'all-providers'
+Requires-Dist: openai>=1.0.0; extra == 'all-providers'
 Provides-Extra: dev
 Requires-Dist: pre-commit>=3.0.0; extra == 'dev'
 Requires-Dist: pytest-asyncio>=0.21.0; extra == 'dev'
@@ -40,6 +43,10 @@ Requires-Dist: mkdocs-material>=9.5.0; extra == 'docs'
 Requires-Dist: mkdocs-minify-plugin>=0.7.0; extra == 'docs'
 Requires-Dist: mkdocs>=1.5.0; extra == 'docs'
 Requires-Dist: mkdocstrings[python]>=0.24.0; extra == 'docs'
+Provides-Extra: gemini
+Requires-Dist: google-generativeai>=0.3.0; extra == 'gemini'
+Provides-Extra: openai
+Requires-Dist: openai>=1.0.0; extra == 'openai'
 Description-Content-Type: text/markdown
 
 # mcpbr

mcpbr-0.6.0/action/README.md

@@ -0,0 +1,46 @@
+# mcpbr GitHub Action
+
+A composite GitHub Action to run MCP server benchmarks in your CI/CD pipeline.
+
+## Usage
+
+```yaml
+- uses: greynewell/mcpbr@main
+  with:
+    config: benchmarks/config.yaml
+    anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }}
+```
+
+## Inputs
+
+| Input | Description | Required | Default |
+|---|---|---|---|
+| `config` | Path to benchmark configuration YAML | Yes | - |
+| `version` | mcpbr version to install | No | `latest` |
+| `python-version` | Python version to use | No | `3.11` |
+| `output-dir` | Directory for results | No | `results` |
+| `extra-args` | Additional CLI arguments | No | `""` |
+| `anthropic-api-key` | Anthropic API key | No | - |
+| `openai-api-key` | OpenAI API key | No | - |
+
+## Outputs
+
+| Output | Description |
+|---|---|
+| `results-path` | Path to the results directory |
+| `exit-code` | Exit code from the benchmark run |
+
+## Examples
+
+See the [examples](examples/) directory for:
+
+- [basic.yml](examples/basic.yml) - Simple benchmark run on push
+- [matrix.yml](examples/matrix.yml) - Matrix builds across benchmarks and Python versions
+
+## Security
+
+Always use GitHub Secrets for API keys. Never hardcode them in workflow files.
+
+```yaml
+anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }}
+```

mcpbr-0.6.0/action/action.yml

@@ -0,0 +1,90 @@
+name: "mcpbr Benchmark Runner"
+description: "Run MCP server benchmarks in your CI/CD pipeline using mcpbr"
+author: "mcpbr Contributors"
+
+branding:
+  icon: "bar-chart-2"
+  color: "blue"
+
+inputs:
+  config:
+    description: "Path to benchmark configuration YAML file"
+    required: true
+  version:
+    description: "Version of mcpbr to install (e.g., '0.6.0' or 'latest')"
+    required: false
+    default: "latest"
+  python-version:
+    description: "Python version to use"
+    required: false
+    default: "3.11"
+  output-dir:
+    description: "Directory for benchmark results"
+    required: false
+    default: "results"
+  extra-args:
+    description: "Additional arguments to pass to mcpbr run"
+    required: false
+    default: ""
+  anthropic-api-key:
+    description: "Anthropic API key (prefer using secrets)"
+    required: false
+  openai-api-key:
+    description: "OpenAI API key (prefer using secrets)"
+    required: false
+
+outputs:
+  results-path:
+    description: "Path to the results directory"
+    value: ${{ steps.run-benchmark.outputs.results-path }}
+  exit-code:
+    description: "Exit code from mcpbr run"
+    value: ${{ steps.run-benchmark.outputs.exit-code }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.python-version }}
+
+    - name: Install mcpbr
+      shell: bash
+      run: |
+        python -m pip install --upgrade pip
+        if [ "${{ inputs.version }}" = "latest" ]; then
+          pip install mcpbr
+        else
+          pip install "mcpbr==${{ inputs.version }}"
+        fi
+
+    - name: Create output directory
+      shell: bash
+      run: mkdir -p "${{ inputs.output-dir }}"
+
+    - name: Run benchmarks
+      id: run-benchmark
+      shell: bash
+      env:
+        ANTHROPIC_API_KEY: ${{ inputs.anthropic-api-key }}
+        OPENAI_API_KEY: ${{ inputs.openai-api-key }}
+      run: |
+        set +e
+        mcpbr run \
+          --config "${{ inputs.config }}" \
+          --output-dir "${{ inputs.output-dir }}" \
+          ${{ inputs.extra-args }}
+        EXIT_CODE=$?
+        set -e
+        echo "exit-code=${EXIT_CODE}" >> "$GITHUB_OUTPUT"
+        echo "results-path=${{ inputs.output-dir }}" >> "$GITHUB_OUTPUT"
+        exit ${EXIT_CODE}
+
+    - name: Upload results artifact
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: mcpbr-results
+        path: ${{ inputs.output-dir }}
+        retention-days: 30

mcpbr-0.6.0/action/examples/basic.yml

@@ -0,0 +1,25 @@
+# Basic example: Run mcpbr benchmarks on push to main
+name: MCP Benchmark
+
+on:
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+jobs:
+  benchmark:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Run MCP benchmarks
+        uses: greynewell/mcpbr@main
+        with:
+          config: benchmarks/config.yaml
+          anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }}
+          output-dir: results
+
+      - name: Check results
+        run: |
+          echo "Benchmark results saved to results/"
+          ls -la results/

mcpbr-0.6.0/action/examples/matrix.yml

@@ -0,0 +1,62 @@
+# Matrix example: Run benchmarks across multiple configurations
+name: MCP Benchmark Matrix
+
+on:
+  pull_request:
+    branches: [main]
+  schedule:
+    - cron: "0 6 * * 1"  # Weekly on Monday at 6am UTC
+
+jobs:
+  benchmark:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        benchmark:
+          - swebench
+          - humaneval
+          - mbpp
+        python-version:
+          - "3.11"
+          - "3.12"
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Run ${{ matrix.benchmark }} benchmark
+        uses: greynewell/mcpbr@main
+        with:
+          config: benchmarks/${{ matrix.benchmark }}.yaml
+          python-version: ${{ matrix.python-version }}
+          anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }}
+          output-dir: results-${{ matrix.benchmark }}-py${{ matrix.python-version }}
+
+      - name: Upload individual results
+        uses: actions/upload-artifact@v4
+        with:
+          name: results-${{ matrix.benchmark }}-py${{ matrix.python-version }}
+          path: results-${{ matrix.benchmark }}-py${{ matrix.python-version }}
+
+  aggregate:
+    needs: benchmark
+    runs-on: ubuntu-latest
+    if: always()
+    steps:
+      - name: Download all results
+        uses: actions/download-artifact@v4
+        with:
+          path: all-results
+
+      - name: Summary
+        run: |
+          echo "## Benchmark Results" >> $GITHUB_STEP_SUMMARY
+          for dir in all-results/results-*; do
+            benchmark=$(basename "$dir")
+            echo "### ${benchmark}" >> $GITHUB_STEP_SUMMARY
+            for json_file in "${dir}"/*.json; do
+              if [ -f "$json_file" ]; then
+                cat "$json_file" >> $GITHUB_STEP_SUMMARY
+              fi
+            done
+          done