mcpbr 0.5.1__py3-none-any.whl → 0.5.2__py3-none-any.whl

mcpbr/config.py

@@ -339,6 +339,12 @@ class HarnessConfig(BaseModel):
         description="Timeout for each task in seconds",
     )
 
+    eval_timeout_seconds: int = Field(
+        default=600,
+        description="Timeout for post-agent evaluation (applying patches and running tests) "
+        "in seconds. Set higher for benchmarks with slow test suites (e.g., Django).",
+    )
+
     max_concurrent: int = Field(
         default=4,
         description="Maximum concurrent task evaluations",

mcpbr/docker_env.py

@@ -139,6 +139,7 @@ class TaskEnvironment:
         timeout: int = 60,
         workdir: str | None = None,
         environment: dict[str, str] | None = None,
+        user: str | None = None,
     ) -> tuple[int, str, str]:
         """Execute a command in the container.
 
@@ -147,6 +148,7 @@ class TaskEnvironment:
             timeout: Timeout in seconds.
             workdir: Working directory (defaults to /workspace).
             environment: Optional environment variables to set.
+            user: Optional user to run the command as (e.g., "mcpbr").
 
         Returns:
             Tuple of (exit_code, stdout, stderr).
@@ -164,6 +166,7 @@ class TaskEnvironment:
                 workdir=wd,
                 demux=True,
                 environment=environment,
+                user=user or "",
             )
             stdout = result.output[0].decode("utf-8") if result.output[0] else ""
             stderr = result.output[1].decode("utf-8") if result.output[1] else ""
@@ -438,17 +441,17 @@ CMD ["/bin/bash"]
                     rm=True,
                 )
         except Exception as e:
-            # Last resort: just tag the base image
-            logger.warning(
-                f"Failed to build comprehensive fallback image: {e}. "
-                f"Falling back to tagging python:3.11-slim as {self.FALLBACK_IMAGE}"
+            # Do NOT tag bare python:3.11-slim as the fallback image — it lacks
+            # git, build tools, etc. and will poison all subsequent tasks.
+            # Instead, mark the build as failed so callers get a clear error.
+            logger.error(
+                f"Failed to build fallback image: {e}. "
+                f"Tasks requiring the fallback image will fail."
             )
-            try:
-                img = self.client.images.get("python:3.11-slim")
-                img.tag(self.FALLBACK_IMAGE)
-            except Exception as tag_error:
-                logger.error(f"Failed to create fallback image {self.FALLBACK_IMAGE}: {tag_error}")
-                raise
+            raise RuntimeError(
+                f"Cannot create fallback image '{self.FALLBACK_IMAGE}': {e}. "
+                f"Ensure Docker has enough disk space and network access."
+            ) from e
 
     async def create_environment(
         self,
@@ -482,7 +485,10 @@ CMD ["/bin/bash"]
         self._temp_dirs.append(temp_dir)
         host_workdir = temp_dir.name
 
-        container_name = f"mcpbr-{self._session_id}-{instance_id}"
+        # Use a unique suffix per container to prevent name collisions when
+        # MCP and baseline runs create containers for the same task.
+        unique_suffix = uuid.uuid4().hex[:6]
+        container_name = f"mcpbr-{self._session_id}-{instance_id}-{unique_suffix}"
 
         container_workdir = "/testbed" if uses_prebuilt else "/workspace"
 
@@ -520,18 +526,35 @@ CMD ["/bin/bash"]
                     )
                     return container
                 except docker.errors.APIError as e:
-                    # Only retry on 500 errors (transient Docker daemon issues)
                     response = getattr(e, "response", None)
-                    if response is not None and getattr(response, "status_code", None) == 500:
-                        if attempt < max_retries:
-                            delay = base_delay * (2**attempt)  # Exponential backoff: 1s, 2s, 4s
-                            logger.warning(
-                                f"Docker API error (attempt {attempt + 1}/{max_retries + 1}): {e}. "
-                                f"Retrying in {delay}s..."
-                            )
-                            time.sleep(delay)
-                            continue
-                    # Re-raise for non-500 errors or after max retries
+                    status_code = getattr(response, "status_code", None) if response else None
+
+                    # On 409 Conflict (container name already in use), try to
+                    # remove the stale container and retry once.
+                    if status_code == 409 and attempt < max_retries:
+                        logger.warning(
+                            f"Container name conflict (attempt {attempt + 1}): {e}. "
+                            f"Removing stale container and retrying..."
+                        )
+                        try:
+                            stale = self.client.containers.get(container_name)
+                            stale.remove(force=True)
+                        except Exception:
+                            pass  # Container may already be gone
+                        time.sleep(1)
+                        continue
+
+                    # Retry on 500 errors (transient Docker daemon issues)
+                    if status_code == 500 and attempt < max_retries:
+                        delay = base_delay * (2**attempt)
+                        logger.warning(
+                            f"Docker API error (attempt {attempt + 1}/{max_retries + 1}): {e}. "
+                            f"Retrying in {delay}s..."
+                        )
+                        time.sleep(delay)
+                        continue
+
+                    # Re-raise for unrecoverable errors or after max retries
                     raise
 
         loop = asyncio.get_event_loop()
@@ -584,6 +607,22 @@ CMD ["/bin/bash"]
             timeout=30,
         )
 
+        # Flush filesystem buffers to ensure all copied files are visible
+        # before any subsequent commands (e.g., setup_command) run.
+        await env.exec_command("sync", timeout=10)
+
+        # Verify workspace is populated — catch copy failures early
+        exit_code, stdout, _ = await env.exec_command(
+            "find /workspace -maxdepth 1 -mindepth 1 | head -5 | wc -l",
+            timeout=10,
+        )
+        file_count = int(stdout.strip()) if exit_code == 0 and stdout.strip().isdigit() else 0
+        if file_count == 0:
+            raise RuntimeError(
+                "Workspace /workspace appears empty after copy from /testbed. "
+                "The filesystem may not have synced correctly."
+            )
+
         env.workdir = "/workspace"
 
     async def _install_claude_cli(self, env: TaskEnvironment) -> None:

mcpbr/evaluation.py

@@ -1,6 +1,7 @@
 """Evaluation logic for applying patches and running tests."""
 
 import ast
+import asyncio
 import json
 from dataclasses import dataclass
 from typing import Any
@@ -182,7 +183,7 @@ async def run_tests(
                 }
             )
 
-        except TimeoutError:
+        except (TimeoutError, asyncio.TimeoutError):
             results.append(
                 {
                     "test": test,

mcpbr/harness.py

@@ -466,7 +466,10 @@ async def _run_mcp_evaluation(
             profiler.sample_memory()
 
         if agent_result.patch:
-            eval_result_dict = await benchmark.evaluate(env, task, agent_result.patch)
+            eval_result_dict = await asyncio.wait_for(
+                benchmark.evaluate(env, task, agent_result.patch),
+                timeout=config.eval_timeout_seconds,
+            )
             # Convert benchmark result format to EvaluationResult-like object
             eval_result = dict_to_namespace(eval_result_dict)
         else:
@@ -619,7 +622,10 @@ async def _run_baseline_evaluation(
             profiler.sample_memory()
 
         if agent_result.patch:
-            eval_result_dict = await benchmark.evaluate(env, task, agent_result.patch)
+            eval_result_dict = await asyncio.wait_for(
+                benchmark.evaluate(env, task, agent_result.patch),
+                timeout=config.eval_timeout_seconds,
+            )
             # Convert benchmark result format to EvaluationResult-like object
             eval_result = dict_to_namespace(eval_result_dict)
         else:

mcpbr/harnesses.py

@@ -455,7 +455,8 @@ MCP_PROMPT_SUFFIX = (
     "\n\nYou have access to an MCP server with additional tools for codebase analysis. "
     "Use these tools to understand the codebase structure, find definitions, trace call chains, "
     "and navigate dependencies before making changes. The MCP tools are especially useful for "
-    "understanding how code is connected across files."
+    "understanding how code is connected across files. "
+    "The repository is located at {workdir}."
 )
 
 
@@ -581,18 +582,22 @@ class ClaudeCodeHarness:
         # Source the env file so setup_command has access to API keys etc.
         env_file = "/tmp/.mcpbr_env.sh"
         setup_full_cmd = f"source {shlex.quote(env_file)} && {setup_cmd}"
+
+        # Run as the mcpbr user so files created by setup_command are
+        # accessible to the agent, which also runs as mcpbr.
+        setup_user = "mcpbr" if env.claude_cli_installed else None
+
         setup_exit, _setup_stdout, setup_stderr = await env.exec_command(
             ["/bin/bash", "-c", setup_full_cmd],
             timeout=setup_timeout,
+            user=setup_user,
         )
 
         if setup_exit != 0:
-            if verbose:
-                self._console.print(
-                    f"[yellow]⚠ Setup command exited with code {setup_exit}[/yellow]"
-                )
-                if setup_stderr:
-                    self._console.print(f"[dim]{setup_stderr[:500]}[/dim]")
+            # Always warn on failure so users can diagnose issues (#388)
+            self._console.print(f"[yellow]⚠ Setup command exited with code {setup_exit}[/yellow]")
+            if verbose and setup_stderr:
+                self._console.print(f"[dim]{setup_stderr[:500]}[/dim]")
             # Non-fatal: continue with agent even if setup fails
         elif verbose:
             self._console.print("[green]✓ Setup command completed[/green]")
@@ -633,7 +638,7 @@ class ClaudeCodeHarness:
             )
 
         problem_statement = task.get("problem_statement", "")
-        prompt = self.prompt_template.format(problem_statement=problem_statement)
+        prompt = self.prompt_template.format(problem_statement=problem_statement, workdir=workdir)
         instance_id = task_id or task.get("instance_id", "unknown")
 
         mcp_server_name = None
@@ -798,7 +803,9 @@ class ClaudeCodeHarness:
     ) -> AgentResult:
         """Solve task using Claude Code CLI inside Docker container."""
         problem_statement = task.get("problem_statement", "")
-        prompt = self.prompt_template.format(problem_statement=problem_statement)
+        prompt = self.prompt_template.format(
+            problem_statement=problem_statement, workdir=env.workdir
+        )
         instance_id = task_id or task.get("instance_id", "unknown")
 
         api_key = os.environ.get("ANTHROPIC_API_KEY", "")

{mcpbr-0.5.1.dist-info → mcpbr-0.5.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mcpbr
-Version: 0.5.1
+Version: 0.5.2
 Summary: Model Context Protocol Benchmark Runner - evaluate MCP servers against software engineering benchmarks
 Project-URL: Homepage, https://github.com/greynewell/mcpbr
 Project-URL: Repository, https://github.com/greynewell/mcpbr

{mcpbr-0.5.1.dist-info → mcpbr-0.5.2.dist-info}/RECORD

@@ -3,7 +3,7 @@ mcpbr/__main__.py,sha256=WmeQsAqtW_9tMTNKArH1m76DPBokZpXuy6dMZp13gXA,132
 mcpbr/agent.py,sha256=aSFH2S3ExKZfdVfMbzk6D1nRhpKt4JmpRzmF4Vi6Gmo,5795
 mcpbr/cache.py,sha256=YiP13omwMbXLb6NhNocJvL58enXEx9J8OrvTZnWUkw4,13254
 mcpbr/cli.py,sha256=xvh7gpJx0LzjV3g-Te4FF7BfHubGzDxOiYQsSeQnCEc,68276
-mcpbr/config.py,sha256=7lWV0ZtzyD6WZ07IR4yhT9lyBBPONzlanaO4XHm9OoE,18952
+mcpbr/config.py,sha256=5MaV6jkPYC4t8MvJEGq6VR7bmxcWZ27behLnPAZu3zY,19200
 mcpbr/config_inheritance.py,sha256=0EV9Tv62UFNgZoc8mY7yYjHEbnMM_R5EAhSeuK7ajAA,6617
 mcpbr/config_migration.py,sha256=vTs52uYLO0DusB07nHZT2Y27-_eZdZKhaXYWhDFcnJI,16098
 mcpbr/config_validator.py,sha256=ZMEIeK4y6fSwyY46Xv5dK5v3jM4HDKcYkosnIcn7iyI,20488
@@ -13,17 +13,17 @@ mcpbr/dashboard.py,sha256=wt2A-yFgDvQc94wgPPJlz70gFAkyUi41xgfqPL9xRQY,21884
 mcpbr/dataset_streaming.py,sha256=XwQSdvy97yurlcAC5hUwto8bLuCf2A9FSMcwjTD_Tho,16720
 mcpbr/dataset_versioning.py,sha256=Y_ZSGhl8ihl6Kgee_p7VbkNwGhgwIdMZPlRunvk4knY,7149
 mcpbr/docker_cache.py,sha256=jn_9Ak2d8omNmedSCBwA7wrswtEQvB-Bu8TIP2cm-F0,18704
-mcpbr/docker_env.py,sha256=_45OUZKjUevE9O3YLF_1uvQtdOyJ7yZIYWmSvXN3cFw,31794
+mcpbr/docker_env.py,sha256=dRhQamlEq05h4wOjZN76c0GIYR6FRx9aGB_Jrkmssss,33676
 mcpbr/docker_prewarm.py,sha256=GVRD2B10HA7OpWq_CC7CkNkJ1OUjAU7GzKOpJ5VFrXk,12638
 mcpbr/dry_run.py,sha256=w_1L5K4Bk3SzeXfZY2NDbXims_Qh6711wIGm6p3tr84,18218
 mcpbr/env_expansion.py,sha256=Rkhth-tWV8CptQlSSk9exuMsUaSTTW9hj69z4snZd_U,6122
-mcpbr/evaluation.py,sha256=EjPREWv7hBRqhBhNan0ERh2imqMBegT0Y2cgZlTxRGk,12765
+mcpbr/evaluation.py,sha256=NK_lId2fbmKZiAyalonhCuLY-pGSGy4tPYN-i84sx8Q,12804
 mcpbr/failure_analysis.py,sha256=N5xp9YPe2d7P9fTa2LVSHsPgB1WOQtWMeClq3bOv4_c,19883
 mcpbr/few_shot.py,sha256=bFDdes_kgZAFWoFZQEfZG5Z2Es9rmkB1jsxSMp4aCCM,11684
 mcpbr/formatting.py,sha256=lwZcb4fD5osBzJlerICyvAVb4KHSm_nRTBg1dVfD6Lo,14193
 mcpbr/gpu_support.py,sha256=eroBiLkt1A3Q2ODJDSyqrd_BzcMh8tFkjtPn7PsvJJc,5070
-mcpbr/harness.py,sha256=xfnD4si0DflBor1cfu_4wrCpECJ9_8eudLEsgVCU6Oo,53731
-mcpbr/harnesses.py,sha256=1FmUfFSQF0HBvmJsNEbyW_Km4ChsWhShY70aQP6_TBI,47947
+mcpbr/harness.py,sha256=Ehq-Yxsvi9lWBHEqdhKx1S6LB4vbDttxHB-REcWBoNo,53935
+mcpbr/harnesses.py,sha256=iaGlRIXdvIqCrYQtXNRZT9HowgmPDVssT2_Qlj2eCkI,48294
 mcpbr/incremental_save.py,sha256=1dm3pGiEIhP8cVk_Y6XF_cAdo3B_vyRc6CO8Wt-MyIA,4830
 mcpbr/junit_reporter.py,sha256=M_02zJbFbA3VoIYG5oR7VDecqWHEpIee-JOUShWNuLU,9261
 mcpbr/latency_metrics.py,sha256=xNMaUzGMSbOIfuoyZGyIfyMk5uAmoj6K65ZAs5D6Z8c,10476
@@ -92,15 +92,15 @@ mcpbr/infrastructure/azure_health.py,sha256=xITmIa9IfYIwxcVhY0sJ81a-6WNKiT8kSQTd
 mcpbr/infrastructure/base.py,sha256=Olj6uiNBeGoUqltZI1NHZfa26kzT-6jfp8YIXSykFKM,3037
 mcpbr/infrastructure/local.py,sha256=VK6UAg7Dzvb9v1LAJgNGA_s0blQKrHAQEXBAC75zAL8,4237
 mcpbr/infrastructure/manager.py,sha256=j0T7U1Tbajmfve4SNfhYKikvL9kgSVT01fYKMC-sH-s,4796
-mcpbr-0.5.1.data/data/mcpbr/data/templates/brave-search.yaml,sha256=PYHXJOaDqYKoqdJc3JV1WbaL-BacrdkQPck1eKGbMPo,1098
-mcpbr-0.5.1.data/data/mcpbr/data/templates/filesystem.yaml,sha256=1p6Z6ChViFYHAODYD71JFst6gdhR5y5rnWNf7Pp5zOY,1091
-mcpbr-0.5.1.data/data/mcpbr/data/templates/github.yaml,sha256=uzPwq5_loFegvH6RNov1MQclbBiFBgYWzpiKLfEN9H4,1133
-mcpbr-0.5.1.data/data/mcpbr/data/templates/google-maps.yaml,sha256=ldR7E9UmuAA-3nJZ1SShD7PhG0_AwDJOSYuy19hQ6cI,1116
-mcpbr-0.5.1.data/data/mcpbr/data/templates/postgres.yaml,sha256=r6R1069BhV4ADQGPZ-T9r6xMNwbr2yrNh8-IHPb4XiI,1178
-mcpbr-0.5.1.data/data/mcpbr/data/templates/slack.yaml,sha256=dBn_YqlFJMJai_55sRDb4hXClgxRpcyYTlWl4LBkpuo,1072
-mcpbr-0.5.1.data/data/mcpbr/data/templates/sqlite.yaml,sha256=UR5yN9f8v_BC6oskny2xMldHWzZrB9b_PpFSmv5eccg,1080
-mcpbr-0.5.1.dist-info/METADATA,sha256=1iupVSrsq687pZ0s77Hu5q0aDex74p-x7ODS876ey3E,55068
-mcpbr-0.5.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-mcpbr-0.5.1.dist-info/entry_points.txt,sha256=lLL8icujqBF36V9bF4gfaB2at4cFKCiv2IdJ1i5hT9U,41
-mcpbr-0.5.1.dist-info/licenses/LICENSE,sha256=mcXLPreEXzD-816yLKmocCPr9_k3gFFo62TjrSuKkIQ,1075
-mcpbr-0.5.1.dist-info/RECORD,,
+mcpbr-0.5.2.data/data/mcpbr/data/templates/brave-search.yaml,sha256=PYHXJOaDqYKoqdJc3JV1WbaL-BacrdkQPck1eKGbMPo,1098
+mcpbr-0.5.2.data/data/mcpbr/data/templates/filesystem.yaml,sha256=1p6Z6ChViFYHAODYD71JFst6gdhR5y5rnWNf7Pp5zOY,1091
+mcpbr-0.5.2.data/data/mcpbr/data/templates/github.yaml,sha256=uzPwq5_loFegvH6RNov1MQclbBiFBgYWzpiKLfEN9H4,1133
+mcpbr-0.5.2.data/data/mcpbr/data/templates/google-maps.yaml,sha256=ldR7E9UmuAA-3nJZ1SShD7PhG0_AwDJOSYuy19hQ6cI,1116
+mcpbr-0.5.2.data/data/mcpbr/data/templates/postgres.yaml,sha256=r6R1069BhV4ADQGPZ-T9r6xMNwbr2yrNh8-IHPb4XiI,1178
+mcpbr-0.5.2.data/data/mcpbr/data/templates/slack.yaml,sha256=dBn_YqlFJMJai_55sRDb4hXClgxRpcyYTlWl4LBkpuo,1072
+mcpbr-0.5.2.data/data/mcpbr/data/templates/sqlite.yaml,sha256=UR5yN9f8v_BC6oskny2xMldHWzZrB9b_PpFSmv5eccg,1080
+mcpbr-0.5.2.dist-info/METADATA,sha256=eZ6PAv2rJM_jGEy3zNPiljm3mkOc8kiO28wrWSLCXv8,55068
+mcpbr-0.5.2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+mcpbr-0.5.2.dist-info/entry_points.txt,sha256=lLL8icujqBF36V9bF4gfaB2at4cFKCiv2IdJ1i5hT9U,41
+mcpbr-0.5.2.dist-info/licenses/LICENSE,sha256=mcXLPreEXzD-816yLKmocCPr9_k3gFFo62TjrSuKkIQ,1075
+mcpbr-0.5.2.dist-info/RECORD,,