mcpbr 0.5.3__py3-none-any.whl → 0.5.4__py3-none-any.whl

mcpbr/__init__.py

@@ -3,4 +3,4 @@
 A benchmark runner for evaluating MCP servers against SWE-bench tasks.
 """
 
-__version__ = "0.5.3"
+__version__ = "0.5.4"

mcpbr/evaluation.py

@@ -93,43 +93,52 @@ async def apply_patch(
 
     workdir = workdir or env.workdir
 
-    # Reset repository to clean state before applying patch
-    # The agent modified files directly, so we need to restore HEAD state
-    await env.exec_command("git reset --hard HEAD", timeout=30, workdir=workdir)
-    await env.exec_command("git clean -fd", timeout=30, workdir=workdir)
-
-    await env.write_file("fix.patch", patch, workdir=workdir)
+    # Use longer timeouts for git operations — under concurrent load,
+    # Docker exec can be slow and 30s is insufficient (#399).
+    try:
+        # Reset repository to clean state before applying patch
+        # The agent modified files directly, so we need to restore HEAD state
+        await env.exec_command("git reset --hard HEAD", timeout=120, workdir=workdir)
+        await env.exec_command("git clean -fd", timeout=120, workdir=workdir)
 
-    exit_code, stdout, stderr = await env.exec_command(
-        "git apply --check fix.patch",
-        timeout=30,
-        workdir=workdir,
-    )
+        await env.write_file("fix.patch", patch, workdir=workdir)
 
-    if exit_code != 0:
-        exit_code2, stdout2, stderr2 = await env.exec_command(
-            "git apply --check -3 fix.patch",
-            timeout=30,
-            workdir=workdir,
-        )
-        if exit_code2 != 0:
-            return False, f"Patch does not apply: {stderr or stderr2}"
         exit_code, stdout, stderr = await env.exec_command(
-            "git apply -3 fix.patch",
-            timeout=30,
-            workdir=workdir,
-        )
-    else:
-        exit_code, stdout, stderr = await env.exec_command(
-            "git apply fix.patch",
-            timeout=30,
+            "git apply --check fix.patch",
+            timeout=120,
             workdir=workdir,
         )
 
-    if exit_code != 0:
-        return False, f"Failed to apply patch: {stderr}"
+        if exit_code != 0:
+            exit_code2, stdout2, stderr2 = await env.exec_command(
+                "git apply --check -3 fix.patch",
+                timeout=120,
+                workdir=workdir,
+            )
+            if exit_code2 != 0:
+                return False, f"Patch does not apply: {stderr or stderr2}"
+            exit_code, stdout, stderr = await env.exec_command(
+                "git apply -3 fix.patch",
+                timeout=120,
+                workdir=workdir,
+            )
+        else:
+            exit_code, stdout, stderr = await env.exec_command(
+                "git apply fix.patch",
+                timeout=120,
+                workdir=workdir,
+            )
 
-    return True, ""
+        if exit_code != 0:
+            return False, f"Failed to apply patch: {stderr}"
+
+        return True, ""
+
+    except (TimeoutError, asyncio.TimeoutError):
+        # Catch exec_command timeouts here so they don't bubble up as
+        # asyncio.TimeoutError to the harness, which would misclassify
+        # this as an agent/eval timeout (#399).
+        return False, "Docker exec timed out during patch application"
 
 
 async def run_tests(
@@ -282,38 +291,43 @@ async def _apply_test_patch(
 
     workdir = workdir or env.workdir
 
-    await env.write_file("test.patch", test_patch, workdir=workdir)
-
-    exit_code, stdout, stderr = await env.exec_command(
-        "git apply --check test.patch",
-        timeout=30,
-        workdir=workdir,
-    )
+    try:
+        await env.write_file("test.patch", test_patch, workdir=workdir)
 
-    if exit_code != 0:
         exit_code, stdout, stderr = await env.exec_command(
-            "git apply --check -3 test.patch",
-            timeout=30,
+            "git apply --check test.patch",
+            timeout=120,
             workdir=workdir,
         )
+
+        if exit_code != 0:
+            exit_code, stdout, stderr = await env.exec_command(
+                "git apply --check -3 test.patch",
+                timeout=120,
+                workdir=workdir,
+            )
+            if exit_code != 0:
+                return True, ""
+            exit_code, stdout, stderr = await env.exec_command(
+                "git apply -3 test.patch",
+                timeout=120,
+                workdir=workdir,
+            )
+        else:
+            exit_code, stdout, stderr = await env.exec_command(
+                "git apply test.patch",
+                timeout=120,
+                workdir=workdir,
+            )
+
         if exit_code != 0:
             return True, ""
-        exit_code, stdout, stderr = await env.exec_command(
-            "git apply -3 test.patch",
-            timeout=30,
-            workdir=workdir,
-        )
-    else:
-        exit_code, stdout, stderr = await env.exec_command(
-            "git apply test.patch",
-            timeout=30,
-            workdir=workdir,
-        )
 
-    if exit_code != 0:
         return True, ""
 
-    return True, ""
+    except (TimeoutError, asyncio.TimeoutError):
+        # Don't let exec timeouts bubble up to the harness (#399)
+        return True, ""
 
 
 async def evaluate_patch(
@@ -356,7 +370,14 @@ async def evaluate_patch(
 
     # Skip dependency installation for pre-built images (already done)
     if not env.uses_prebuilt:
-        await _install_dependencies(env)
+        try:
+            await _install_dependencies(env)
+        except (TimeoutError, asyncio.TimeoutError):
+            return EvaluationResult(
+                resolved=False,
+                patch_applied=True,
+                error="Docker exec timed out during dependency installation",
+            )
 
     repo = task.get("repo")
 

mcpbr/harness.py

@@ -1,6 +1,7 @@
 """Main evaluation harness orchestrating parallel task execution."""
 
 import asyncio
+import logging
 import time
 from dataclasses import dataclass
 from datetime import datetime, timezone
@@ -29,6 +30,7 @@ from .pricing import calculate_cost
 from .profiler import PerformanceProfiler
 
 console = Console()
+logger = logging.getLogger(__name__)
 
 
 class SimpleNamespace:
@@ -56,6 +58,57 @@ def dict_to_namespace(data: Any) -> Any:
         return data
 
 
+# -- Cold-start mitigation helpers (#401) ------------------------------------
+
+# Seconds between each task launch in the first concurrent batch.
+_STAGGER_INTERVAL = 1.0
+
+
+def _stagger_delay(task_index: int, max_concurrent: int) -> float:
+    """Return the startup delay for a task to avoid cold-start contention.
+
+    Only the first batch (indices 0 .. max_concurrent-1) is staggered.
+    The very first task starts immediately; subsequent tasks in the batch
+    get an increasing delay so Docker image pulls and container creation
+    don't all hit at once.
+
+    Args:
+        task_index: Zero-based index of the task in launch order.
+        max_concurrent: Semaphore size / max parallelism.
+
+    Returns:
+        Delay in seconds (0.0 means start immediately).
+    """
+    if max_concurrent <= 1:
+        return 0.0
+    # Only stagger the first batch
+    if task_index >= max_concurrent:
+        return 0.0
+    return task_index * _STAGGER_INTERVAL
+
+
+def _should_retry_zero_iteration(result: dict[str, Any]) -> bool:
+    """Check whether a task result indicates a cold-start failure worth retrying.
+
+    A cold-start failure is characterised by zero iterations AND zero tokens
+    AND a timeout status — the agent never actually ran.
+
+    Args:
+        result: Single-run result dict from _run_mcp_evaluation or _run_baseline_evaluation.
+
+    Returns:
+        True if the result looks like a cold-start failure.
+    """
+    if result.get("status") != "timeout":
+        return False
+    if result.get("iterations", -1) != 0:
+        return False
+    tokens = result.get("tokens", {})
+    if tokens.get("input", -1) != 0 or tokens.get("output", -1) != 0:
+        return False
+    return True
+
+
 @dataclass
 class TaskResult:
     """Result for a single task."""
@@ -302,6 +355,24 @@ async def run_single_task(
                     mcp_server_config=config.mcp_server_a,
                     server_name="server_a",
                 )
+                # Retry once on cold-start failure (#401)
+                if result.mcp_server_a and _should_retry_zero_iteration(result.mcp_server_a):
+                    logger.info(
+                        "Retrying MCP server_a task %s (zero-iteration cold-start)", instance_id
+                    )
+                    result.mcp_server_a = await _run_mcp_evaluation(
+                        task,
+                        config,
+                        docker_manager,
+                        benchmark,
+                        verbose,
+                        verbosity,
+                        mcp_log_writer_a if mcp_log_writer_a else log_file,
+                        cache,
+                        mcp_logs_dir,
+                        mcp_server_config=config.mcp_server_a,
+                        server_name="server_a",
+                    )
             finally:
                 if mcp_log_writer_a:
                     mcp_log_writer_a.close()
@@ -324,6 +395,24 @@ async def run_single_task(
                     mcp_server_config=config.mcp_server_b,
                     server_name="server_b",
                 )
+                # Retry once on cold-start failure (#401)
+                if result.mcp_server_b and _should_retry_zero_iteration(result.mcp_server_b):
+                    logger.info(
+                        "Retrying MCP server_b task %s (zero-iteration cold-start)", instance_id
+                    )
+                    result.mcp_server_b = await _run_mcp_evaluation(
+                        task,
+                        config,
+                        docker_manager,
+                        benchmark,
+                        verbose,
+                        verbosity,
+                        mcp_log_writer_b if mcp_log_writer_b else log_file,
+                        cache,
+                        mcp_logs_dir,
+                        mcp_server_config=config.mcp_server_b,
+                        server_name="server_b",
+                    )
             finally:
                 if mcp_log_writer_b:
                     mcp_log_writer_b.close()
@@ -344,6 +433,20 @@ async def run_single_task(
                     cache,
                     mcp_logs_dir,
                 )
+                # Retry once on cold-start failure (#401)
+                if result.mcp and _should_retry_zero_iteration(result.mcp):
+                    logger.info("Retrying MCP task %s (zero-iteration cold-start)", instance_id)
+                    result.mcp = await _run_mcp_evaluation(
+                        task,
+                        config,
+                        docker_manager,
+                        benchmark,
+                        verbose,
+                        verbosity,
+                        mcp_log_writer if mcp_log_writer else log_file,
+                        cache,
+                        mcp_logs_dir,
+                    )
             finally:
                 if mcp_log_writer:
                     mcp_log_writer.close()
@@ -363,6 +466,19 @@ async def run_single_task(
                 baseline_log_writer if baseline_log_writer else log_file,
                 cache,
             )
+            # Retry once on cold-start failure (#401)
+            if result.baseline and _should_retry_zero_iteration(result.baseline):
+                logger.info("Retrying baseline task %s (zero-iteration cold-start)", instance_id)
+                result.baseline = await _run_baseline_evaluation(
+                    task,
+                    config,
+                    docker_manager,
+                    benchmark,
+                    verbose,
+                    verbosity,
+                    baseline_log_writer if baseline_log_writer else log_file,
+                    cache,
+                )
         finally:
             if baseline_log_writer:
                 baseline_log_writer.close()
@@ -539,7 +655,15 @@ async def _run_mcp_evaluation(
         if env:
             # Track Docker teardown time
             teardown_start = time.time()
-            await env.cleanup()
+            try:
+                await asyncio.wait_for(env.cleanup(), timeout=60)
+            except (asyncio.TimeoutError, Exception) as cleanup_err:
+                logger.warning("Container cleanup failed for MCP task: %s", cleanup_err)
+                try:
+                    if hasattr(env, "container") and env.container:
+                        env.container.remove(force=True)
+                except Exception:
+                    pass
             if profiler:
                 teardown_end = time.time()
                 profiler.record_docker_teardown(teardown_end - teardown_start)
@@ -695,7 +819,15 @@ async def _run_baseline_evaluation(
         if env:
             # Track Docker teardown time
             teardown_start = time.time()
-            await env.cleanup()
+            try:
+                await asyncio.wait_for(env.cleanup(), timeout=60)
+            except (asyncio.TimeoutError, Exception) as cleanup_err:
+                logger.warning("Container cleanup failed for baseline task: %s", cleanup_err)
+                try:
+                    if hasattr(env, "container") and env.container:
+                        env.container.remove(force=True)
+                except Exception:
+                    pass
             if profiler:
                 teardown_end = time.time()
                 profiler.record_docker_teardown(teardown_end - teardown_start)
@@ -1013,9 +1145,10 @@ async def run_evaluation(
     semaphore = asyncio.Semaphore(config.max_concurrent)
     budget_exceeded = False
     current_cost = 0.0
+    _task_launch_counter = 0
 
     async def run_with_semaphore(task: dict[str, Any]) -> TaskResult | None:
-        nonlocal current_cost, budget_exceeded
+        nonlocal current_cost, budget_exceeded, _task_launch_counter
 
         # Check budget before running task
         if config.budget and current_cost >= config.budget:
@@ -1023,6 +1156,15 @@ async def run_evaluation(
             return None
 
         async with semaphore:
+            # Stagger first-batch launches to avoid cold-start contention (#401).
+            # Delay is inside the semaphore so the sleeping task holds its slot
+            # and later tasks cannot leapfrog ahead of the first batch.
+            my_index = _task_launch_counter
+            _task_launch_counter += 1
+            delay = _stagger_delay(my_index, config.max_concurrent)
+            if delay > 0:
+                await asyncio.sleep(delay)
+
             result = await run_single_task(
                 task,
                 config,

{mcpbr-0.5.3.dist-info → mcpbr-0.5.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mcpbr
-Version: 0.5.3
+Version: 0.5.4
 Summary: Model Context Protocol Benchmark Runner - evaluate MCP servers against software engineering benchmarks
 Project-URL: Homepage, https://github.com/greynewell/mcpbr
 Project-URL: Repository, https://github.com/greynewell/mcpbr

{mcpbr-0.5.3.dist-info → mcpbr-0.5.4.dist-info}/RECORD

@@ -1,4 +1,4 @@
-mcpbr/__init__.py,sha256=vWZ1BylD_FMXNCo_5ZWJq6kIJCv49VUU5O4I0Orpko8,151
+mcpbr/__init__.py,sha256=BvGCejBsCXoZ7xBuf4RkvHWegXxymyexsnWe_h-GjwI,151
 mcpbr/__main__.py,sha256=WmeQsAqtW_9tMTNKArH1m76DPBokZpXuy6dMZp13gXA,132
 mcpbr/agent.py,sha256=aSFH2S3ExKZfdVfMbzk6D1nRhpKt4JmpRzmF4Vi6Gmo,5795
 mcpbr/cache.py,sha256=YiP13omwMbXLb6NhNocJvL58enXEx9J8OrvTZnWUkw4,13254
@@ -17,12 +17,12 @@ mcpbr/docker_env.py,sha256=dRhQamlEq05h4wOjZN76c0GIYR6FRx9aGB_Jrkmssss,33676
 mcpbr/docker_prewarm.py,sha256=GVRD2B10HA7OpWq_CC7CkNkJ1OUjAU7GzKOpJ5VFrXk,12638
 mcpbr/dry_run.py,sha256=w_1L5K4Bk3SzeXfZY2NDbXims_Qh6711wIGm6p3tr84,18218
 mcpbr/env_expansion.py,sha256=Rkhth-tWV8CptQlSSk9exuMsUaSTTW9hj69z4snZd_U,6122
-mcpbr/evaluation.py,sha256=NK_lId2fbmKZiAyalonhCuLY-pGSGy4tPYN-i84sx8Q,12804
+mcpbr/evaluation.py,sha256=UbECTCxbUh0dLGmcYWVQdZjwtyyVe3lNMJgbCBa1858,13923
 mcpbr/failure_analysis.py,sha256=N5xp9YPe2d7P9fTa2LVSHsPgB1WOQtWMeClq3bOv4_c,19883
 mcpbr/few_shot.py,sha256=bFDdes_kgZAFWoFZQEfZG5Z2Es9rmkB1jsxSMp4aCCM,11684
 mcpbr/formatting.py,sha256=lwZcb4fD5osBzJlerICyvAVb4KHSm_nRTBg1dVfD6Lo,14193
 mcpbr/gpu_support.py,sha256=eroBiLkt1A3Q2ODJDSyqrd_BzcMh8tFkjtPn7PsvJJc,5070
-mcpbr/harness.py,sha256=Ehq-Yxsvi9lWBHEqdhKx1S6LB4vbDttxHB-REcWBoNo,53935
+mcpbr/harness.py,sha256=LO5viFF5uSfbYriCnIfww598ashHyN_sDT-D0ELN3dY,59999
 mcpbr/harnesses.py,sha256=iaGlRIXdvIqCrYQtXNRZT9HowgmPDVssT2_Qlj2eCkI,48294
 mcpbr/incremental_save.py,sha256=1dm3pGiEIhP8cVk_Y6XF_cAdo3B_vyRc6CO8Wt-MyIA,4830
 mcpbr/junit_reporter.py,sha256=M_02zJbFbA3VoIYG5oR7VDecqWHEpIee-JOUShWNuLU,9261
@@ -92,15 +92,15 @@ mcpbr/infrastructure/azure_health.py,sha256=xITmIa9IfYIwxcVhY0sJ81a-6WNKiT8kSQTd
 mcpbr/infrastructure/base.py,sha256=Olj6uiNBeGoUqltZI1NHZfa26kzT-6jfp8YIXSykFKM,3037
 mcpbr/infrastructure/local.py,sha256=VK6UAg7Dzvb9v1LAJgNGA_s0blQKrHAQEXBAC75zAL8,4237
 mcpbr/infrastructure/manager.py,sha256=j0T7U1Tbajmfve4SNfhYKikvL9kgSVT01fYKMC-sH-s,4796
-mcpbr-0.5.3.data/data/mcpbr/data/templates/brave-search.yaml,sha256=PYHXJOaDqYKoqdJc3JV1WbaL-BacrdkQPck1eKGbMPo,1098
-mcpbr-0.5.3.data/data/mcpbr/data/templates/filesystem.yaml,sha256=1p6Z6ChViFYHAODYD71JFst6gdhR5y5rnWNf7Pp5zOY,1091
-mcpbr-0.5.3.data/data/mcpbr/data/templates/github.yaml,sha256=uzPwq5_loFegvH6RNov1MQclbBiFBgYWzpiKLfEN9H4,1133
-mcpbr-0.5.3.data/data/mcpbr/data/templates/google-maps.yaml,sha256=ldR7E9UmuAA-3nJZ1SShD7PhG0_AwDJOSYuy19hQ6cI,1116
-mcpbr-0.5.3.data/data/mcpbr/data/templates/postgres.yaml,sha256=r6R1069BhV4ADQGPZ-T9r6xMNwbr2yrNh8-IHPb4XiI,1178
-mcpbr-0.5.3.data/data/mcpbr/data/templates/slack.yaml,sha256=dBn_YqlFJMJai_55sRDb4hXClgxRpcyYTlWl4LBkpuo,1072
-mcpbr-0.5.3.data/data/mcpbr/data/templates/sqlite.yaml,sha256=UR5yN9f8v_BC6oskny2xMldHWzZrB9b_PpFSmv5eccg,1080
-mcpbr-0.5.3.dist-info/METADATA,sha256=YJ05sM1v6ApgK9HD6wMn5mIUF96cKUbU4C8nJwwPSgQ,55068
-mcpbr-0.5.3.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-mcpbr-0.5.3.dist-info/entry_points.txt,sha256=lLL8icujqBF36V9bF4gfaB2at4cFKCiv2IdJ1i5hT9U,41
-mcpbr-0.5.3.dist-info/licenses/LICENSE,sha256=mcXLPreEXzD-816yLKmocCPr9_k3gFFo62TjrSuKkIQ,1075
-mcpbr-0.5.3.dist-info/RECORD,,
+mcpbr-0.5.4.data/data/mcpbr/data/templates/brave-search.yaml,sha256=PYHXJOaDqYKoqdJc3JV1WbaL-BacrdkQPck1eKGbMPo,1098
+mcpbr-0.5.4.data/data/mcpbr/data/templates/filesystem.yaml,sha256=1p6Z6ChViFYHAODYD71JFst6gdhR5y5rnWNf7Pp5zOY,1091
+mcpbr-0.5.4.data/data/mcpbr/data/templates/github.yaml,sha256=uzPwq5_loFegvH6RNov1MQclbBiFBgYWzpiKLfEN9H4,1133
+mcpbr-0.5.4.data/data/mcpbr/data/templates/google-maps.yaml,sha256=ldR7E9UmuAA-3nJZ1SShD7PhG0_AwDJOSYuy19hQ6cI,1116
+mcpbr-0.5.4.data/data/mcpbr/data/templates/postgres.yaml,sha256=r6R1069BhV4ADQGPZ-T9r6xMNwbr2yrNh8-IHPb4XiI,1178
+mcpbr-0.5.4.data/data/mcpbr/data/templates/slack.yaml,sha256=dBn_YqlFJMJai_55sRDb4hXClgxRpcyYTlWl4LBkpuo,1072
+mcpbr-0.5.4.data/data/mcpbr/data/templates/sqlite.yaml,sha256=UR5yN9f8v_BC6oskny2xMldHWzZrB9b_PpFSmv5eccg,1080
+mcpbr-0.5.4.dist-info/METADATA,sha256=3bZ7iyaLkIRs3-e6EpOVwfMkvSVStjW4GDKNYKJ9xfM,55068
+mcpbr-0.5.4.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+mcpbr-0.5.4.dist-info/entry_points.txt,sha256=lLL8icujqBF36V9bF4gfaB2at4cFKCiv2IdJ1i5hT9U,41
+mcpbr-0.5.4.dist-info/licenses/LICENSE,sha256=mcXLPreEXzD-816yLKmocCPr9_k3gFFo62TjrSuKkIQ,1075
+mcpbr-0.5.4.dist-info/RECORD,,