featurebench 0.2.2__tar.gz → 0.2.3__tar.gz

{featurebench-0.2.2 → featurebench-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: featurebench
-Version: 0.2.2
+Version: 0.2.3
 Summary: FeatureBench Pipeline - A test-driven data generation pipeline for building and evaluating feature-level coding benchmarks
 License-File: LICENSE
 Requires-Python: >=3.12
@@ -52,7 +52,7 @@ It provides a unified CLI to run inference, evaluation, and dataset generation.
 <!-- LEADERBOARD_START -->
 ## 🏆 Leaderboard
 
-[Full interactive leaderboard](https://libercoders.github.io/FeatureBench/leaderboard/) with tabs, filters, and sorting.
+[Full interactive leaderboard](https://libercoders.github.io/FeatureBench/leaderboard/) with tabs, filters, and sorting (Dataset V1.0).
 
 <details open>
 <summary><b>Lite split results, ranked by %PASSED</b></summary>
@@ -159,4 +159,4 @@ If you found FeatureBench useful, please cite us as:
 
 ## 📧 Contact
 
-If you have any questions, feel free to contact [zhouqixing@stu.cqu.edu.cn](mailto:zhouqixing@stu.cqu.edu.cn) or [zjcheng2022@gmail.com](mailto:zjcheng2022@gmail.com).
+If you have any questions, feel free to contact [qixing.zhou@smail.nju.edu.cn](mailto:qixing.zhou@smail.nju.edu.cn) or [zjcheng2022@gmail.com](mailto:zjcheng2022@gmail.com).

{featurebench-0.2.2 → featurebench-0.2.3}/README.md

@@ -28,7 +28,7 @@ It provides a unified CLI to run inference, evaluation, and dataset generation.
 <!-- LEADERBOARD_START -->
 ## 🏆 Leaderboard
 
-[Full interactive leaderboard](https://libercoders.github.io/FeatureBench/leaderboard/) with tabs, filters, and sorting.
+[Full interactive leaderboard](https://libercoders.github.io/FeatureBench/leaderboard/) with tabs, filters, and sorting (Dataset V1.0).
 
 <details open>
 <summary><b>Lite split results, ranked by %PASSED</b></summary>
@@ -135,4 +135,4 @@ If you found FeatureBench useful, please cite us as:
 
 ## 📧 Contact
 
-If you have any questions, feel free to contact [zhouqixing@stu.cqu.edu.cn](mailto:zhouqixing@stu.cqu.edu.cn) or [zjcheng2022@gmail.com](mailto:zjcheng2022@gmail.com).
+If you have any questions, feel free to contact [qixing.zhou@smail.nju.edu.cn](mailto:qixing.zhou@smail.nju.edu.cn) or [zjcheng2022@gmail.com](mailto:zjcheng2022@gmail.com).

{featurebench-0.2.2 → featurebench-0.2.3}/config_example.toml

@@ -52,7 +52,8 @@ base_url = "http://localhost:8080/v1"                # vLLM server URL
 # Optional: Host cache directory for agent downloads; will be mounted to /download inside containers
 # Set to empty to disable cache mount (each container will re-download/install)
 # Use an absolute path here
-download_cache_dir="/path/to/FeatureBench/download_cache"
+# Optional shared install cache. Leave unset for leakage-resistant evaluations.
+# download_cache_dir="/path/to/FeatureBench/download_cache"
 
 # OpenHands Agent Configuration
 [infer_config.openhands]

{featurebench-0.2.2 → featurebench-0.2.3}/docs/config.md

@@ -68,10 +68,11 @@ How infer reads `config.toml`:
 
 ```toml
 [infer]
-download_cache_dir = "/abs/path/to/FeatureBench/download_cache"
+# Optional shared install cache. Leave unset for leakage-resistant evaluations.
+# download_cache_dir = "/abs/path/to/FeatureBench/download_cache"
 ```
 
-- `download_cache_dir`: host-side cache directory that will be mounted into the container at `/download`.
+- `download_cache_dir`: optional host-side cache directory mounted into the container at `/download`. It is disabled by default because the agent can read shared cache contents during inference.
   - Purpose: speed up agent installs/downloads (e.g. npm cache).
   - Empty means no mount (each container downloads on its own; usually slower).
 

{featurebench-0.2.2 → featurebench-0.2.3}/docs/infer_cli_arg.md

@@ -160,6 +160,13 @@ flags can override metadata (see the argument list below).
   Default: no override (OpenHands default applies).  
   Resume mode: ignored (uses metadata).
 
+### mini-swe-agent Only
+
+- `--cost-limit`
+  Per-task cost limit in USD, passed to mini-swe-agent.
+  Set to `0` to disable the limit. If omitted, mini-swe-agent's upstream default applies.
+  Resume mode: ignored (uses metadata).
+
 ## Section 4: Output Directory Structure
 
 ```

featurebench-0.2.3/featurebench/environment_fixes/__init__.py

@@ -0,0 +1,13 @@
+"""Repository-specific fixes for leaked or duplicated task-environment artifacts.
+
+Add a fix by implementing :class:`EnvironmentFix`, decorating it with
+``register_environment_fix``, and importing its module below. Each plugin is
+matched by repository and is shared by inference and Oracle evaluation.
+"""
+
+from featurebench.environment_fixes.registry import apply_environment_fixes
+
+# Import built-in plugins so they register themselves.
+from featurebench.environment_fixes import mlflow as _mlflow  # noqa: F401, E402
+
+__all__ = ["apply_environment_fixes"]

featurebench-0.2.3/featurebench/environment_fixes/base.py

@@ -0,0 +1,24 @@
+"""Interfaces for repository-specific task-environment fixes."""
+
+from abc import ABC, abstractmethod
+from collections.abc import Callable
+import logging
+
+
+CommandRunner = Callable[[str], tuple[int, str | bytes]]
+
+
+class EnvironmentFix(ABC):
+    """A targeted cleanup applied before the task repository becomes the Git baseline."""
+
+    name: str
+    repositories: frozenset[str]
+
+    def applies_to(self, repository: str) -> bool:
+        """Return whether this fix applies to a canonical ``owner/repo`` name."""
+        normalized = repository.strip().casefold()
+        return normalized in {repo.casefold() for repo in self.repositories}
+
+    @abstractmethod
+    def apply(self, run_command: CommandRunner, logger: logging.Logger) -> bool:
+        """Apply and verify the fix, returning ``False`` on failure."""

featurebench-0.2.3/featurebench/environment_fixes/mlflow.py

@@ -0,0 +1,41 @@
+"""MLflow-specific task-environment cleanup."""
+
+import logging
+
+from featurebench.environment_fixes.base import CommandRunner, EnvironmentFix
+from featurebench.environment_fixes.registry import register_environment_fix
+
+
+@register_environment_fix
+class MlflowParallelSourcesFix(EnvironmentFix):
+    """Remove unmasked MLflow package copies shipped in the monorepo's ``libs`` tree."""
+
+    name = "mlflow_parallel_sources"
+    repositories = frozenset({"mlflow/mlflow"})
+    paths = (
+        "/testbed/libs/skinny/mlflow",
+        "/testbed/libs/tracing/mlflow",
+    )
+
+    def apply(self, run_command: CommandRunner, logger: logging.Logger) -> bool:
+        paths = " ".join(self.paths)
+        command = f"""
+        paths=({paths})
+        for path in "${{paths[@]}}"; do
+            if [ -e "$path" ] || [ -L "$path" ]; then
+                rm -rf -- "$path" || exit 1
+                echo "Removed MLflow parallel source: $path"
+            fi
+            if [ -e "$path" ] || [ -L "$path" ]; then
+                echo "MLflow parallel source still exists: $path" >&2
+                exit 1
+            fi
+        done
+        """
+        exit_code, output = run_command(command)
+        if exit_code != 0:
+            if isinstance(output, bytes):
+                output = output.decode("utf-8", errors="replace")
+            logger.error("Failed to remove MLflow parallel sources: %s", output)
+            return False
+        return True

featurebench-0.2.3/featurebench/environment_fixes/registry.py

@@ -0,0 +1,43 @@
+"""Registry and execution helpers for task-environment fixes."""
+
+import logging
+from typing import TypeVar
+
+from featurebench.environment_fixes.base import CommandRunner, EnvironmentFix
+
+
+_FIXES: list[EnvironmentFix] = []
+_EnvironmentFixType = TypeVar("_EnvironmentFixType", bound=type[EnvironmentFix])
+
+
+def register_environment_fix(fix_type: _EnvironmentFixType) -> _EnvironmentFixType:
+    """Register an environment-fix class at import time."""
+    fix = fix_type()
+    if any(existing.name == fix.name for existing in _FIXES):
+        raise ValueError(f"Duplicate environment fix name: {fix.name}")
+    _FIXES.append(fix)
+    return fix_type
+
+
+def apply_environment_fixes(
+    repository: str,
+    run_command: CommandRunner,
+    logger: logging.Logger,
+) -> bool:
+    """Apply every registered fix matching ``repository`` in registration order."""
+    matching_fixes = [fix for fix in _FIXES if fix.applies_to(repository)]
+    if not matching_fixes:
+        return True
+
+    for fix in matching_fixes:
+        logger.info("Applying environment fix %s for %s", fix.name, repository)
+        try:
+            if not fix.apply(run_command, logger):
+                logger.error("Environment fix %s failed for %s", fix.name, repository)
+                return False
+        except Exception:
+            logger.exception("Environment fix %s raised for %s", fix.name, repository)
+            return False
+        logger.info("Environment fix %s completed for %s", fix.name, repository)
+
+    return True

{featurebench-0.2.2 → featurebench-0.2.3}/featurebench/harness/runtime.py

@@ -13,6 +13,7 @@ from typing import Any
 import pandas as pd
 from docker.models.containers import Container
 
+from featurebench.environment_fixes import apply_environment_fixes
 from featurebench.harness.constants import (
     APPLY_PATCH_FAIL,
     APPLY_PATCH_PASS,
@@ -55,9 +56,11 @@ def run_instance_level1(
     1. Activate conda environment
     2. Restore project from /root/my_repo/
     3. Apply patch (for masking) and directly delete F2P test files
-    4. Reinitialize git
-    5. Apply agent's patch
-    6. Delete any generated F2P files, restore F2P test, and run tests
+    4. Apply repository-specific environment fixes
+    5. Reinitialize git
+    6. Apply agent's patch
+    7. Delete any generated F2P files and restore the F2P test
+    8. Run tests
 
     Args:
         instance: Instance data (from HuggingFace)
@@ -177,8 +180,20 @@ def run_instance_level1(
             else:
                 logger.warning("No FAIL_TO_PASS tests found in instance, skipping test file deletion")
 
-        # Step 3: Reinitialize git repository
-        logger.info("Step 3: Reinitializing git repository")
+        # Step 3: Apply the same repository-specific cleanup used during
+        # inference. Running it before Git initialization keeps the cleanup out
+        # of the candidate patch while making Oracle validation representative.
+        logger.info("Step 3: Applying repository-specific environment fixes")
+        if not apply_environment_fixes(
+            str(repo_name),
+            lambda command: exec_run_with_timeout(container, command, timeout=120),
+            logger,
+        ):
+            results["error"] = "Repository-specific environment fix failed"
+            return results
+
+        # Step 4: Reinitialize git repository
+        logger.info("Step 4: Reinitializing git repository")
         git_cmds = [
             "cd /testbed && rm -rf .git",
             "cd /testbed && git init",
@@ -194,8 +209,8 @@ def run_instance_level1(
                 logger.warning(f"Git command failed: {cmd}")
                 logger.warning(f"Output: {output.decode(UTF8, errors='replace')}")
 
-        # Step 4: Apply agent's patch
-        logger.info("Step 4: Applying agent patch")
+        # Step 5: Apply agent's patch
+        logger.info("Step 5: Applying agent patch")
         patch_content = pred[KEY_PREDICTION]
 
         if not patch_content or patch_content.strip() == "":
@@ -231,8 +246,8 @@ def run_instance_level1(
         logger.info(f"{APPLY_PATCH_PASS}")
         results["patch_applied"] = True
 
-        # Step 5: Prepare for testing - delete generated F2P files and restore F2P test
-        logger.info("Step 5: Preparing for testing - cleaning up and restoring F2P test")
+        # Step 6: Prepare for testing - delete generated F2P files and restore F2P test
+        logger.info("Step 6: Preparing for testing - cleaning up and restoring F2P test")
 
         fail_to_pass = instance.get('FAIL_TO_PASS', [])
         if not fail_to_pass:
@@ -268,8 +283,8 @@ def run_instance_level1(
         else:
             logger.warning("No test_patch available to restore F2P test file")
 
-        # Step 6: Run tests
-        logger.info("Step 6: Running tests")
+        # Step 7: Run tests
+        logger.info("Step 7: Running tests")
 
         # Get test configuration from repo_settings
         repo_settings = parse_repo_settings(instance)
@@ -589,4 +604,3 @@ def run_instance_level2(
         logger.error(traceback.format_exc())
         results["error"] = str(e)
         return results
-

{featurebench-0.2.2 → featurebench-0.2.3}/featurebench/infer/agents/base.py

@@ -41,6 +41,10 @@ class BaseAgent(ABC):
         self.env_vars = env_vars or {}
         self.logger = logger or logging.getLogger(__name__)
         self._kwargs = kwargs
+        # Agent adapters may populate this with their native terminal status.
+        # It deliberately remains optional because not every supported agent
+        # exposes a structured status artifact.
+        self.agent_exit_status: Optional[str] = None
     
     @property
     @abstractmethod
@@ -354,6 +358,7 @@ class BaseAgent(ABC):
         Returns:
             True if agent completed successfully
         """
+        self.agent_exit_status = None
         self.logger.info(f"Running {self.name} agent...")
         
         with open(log_file, "a", encoding="utf-8") as f:
@@ -385,16 +390,23 @@ class BaseAgent(ABC):
                 log_file=log_file,
                 timeout=timeout
             )
-            
+
             success_run = exit_code == 0
+            timed_out = not success_run and self._infer_log_has_timeout_marker(log_file)
             success_post_run = self.post_run_hook(container, log_file)
 
+            # A host-side timeout may kill the agent before it can persist a
+            # native terminal status. Preserve the actual reason for stopping
+            # instead of reporting an unknown/generic failure.
+            if timed_out:
+                self.agent_exit_status = "TimeExceeded"
+
             if success_run and success_post_run:
                 self.logger.info(f"{self.name} agent completed successfully")
                 return True
 
             if not success_run:
-                if self._force_timeout_enabled() and self._infer_log_has_timeout_marker(log_file):
+                if self._force_timeout_enabled() and timed_out:
                     post_run_state = "passed" if success_post_run else "failed"
                     self.logger.warning(
                         f"{self.name} run hit timeout marker (post-run hook {post_run_state}); "

{featurebench-0.2.2 → featurebench-0.2.3}/featurebench/infer/agents/mini_swe_agent.py

@@ -4,6 +4,7 @@ mini-swe-agent implementation.
 This agent runs the external `mini` CLI in the container.
 """
 
+import json
 import shlex
 from pathlib import Path
 from typing import Dict, List, Optional
@@ -76,7 +77,7 @@ set -e
 echo "Installing mini-swe-agent..."
 
 apt-get update
-apt-get install -y python3 python3-pip python3-venv
+apt-get install -y passwd python3 python3-pip python3-venv util-linux
 
 CACHE_ROOT="${{AGENT_DOWNLOAD_CACHE:-/download}}"
 mkdir -p "$CACHE_ROOT" "$CACHE_ROOT/pip"
@@ -98,29 +99,72 @@ echo "mini-swe-agent installation complete"
         self.cm.exec_command(container, "mkdir -p /agent-logs", log_file=log_file)
         return True
 
-    def post_run_hook(self, container, log_file) -> bool:
-        """Persist mini-swe-agent CLI output (best-effort)."""
-        log_dir = Path(log_file).parent
+    def pre_run_setup(self, container, instance, log_file: Path) -> bool:
+        """Isolate agent-issued shell commands from controller dependencies."""
+        controller_dir = "/opt/featurebench-controller"
+        environment_source = (
+            Path(__file__).parent.parent / "environments" / "mini_swe.py"
+        )
+        environment_target = f"{controller_dir}/featurebench_mini_swe_environment.py"
+
         try:
-            self.cm.copy_from_container(
+            exit_code, output = self.cm.exec_command(
                 container,
-                "/agent-logs/mini_swe_agent_output.log",
-                log_dir / "mini_swe_agent_output.log",
+                "set -e\n"
+                "getent group fbagent >/dev/null || groupadd --system fbagent\n"
+                "id -u fbagent >/dev/null 2>&1 || "
+                "useradd --system --gid fbagent --home-dir /home/fbagent "
+                "--create-home --shell /bin/bash fbagent\n"
+                f"install -d -o root -g root -m 700 {controller_dir} "
+                "/agent-logs\n",
+                log_file=log_file,
             )
-        except Exception:
-            pass
-        try:
-            self.cm.copy_from_container(
+            if exit_code != 0:
+                raise RuntimeError(
+                    f"Failed to create restricted mini-swe-agent user: {output}"
+                )
+
+            self.cm.copy_to_container(
                 container,
-                "/root/.config/mini-swe-agent/last_mini_run.traj.json",
-                log_dir / "traj.json",
+                environment_source,
+                environment_target,
             )
-        except Exception:
-            pass
+            exit_code, output = self.cm.exec_command(
+                container,
+                "set -e\n"
+                f"chown root:root {environment_target}\n"
+                f"chmod 600 {environment_target}\n"
+                "chown -R fbagent:fbagent /testbed\n"
+                "chmod 700 /opt/mini-swe-agent-venv\n"
+                f"chmod 700 {controller_dir} /installed-agent /agent-logs\n"
+                "setpriv --reuid=fbagent --regid=fbagent --init-groups "
+                "--no-new-privs /bin/bash -c "
+                "'test -r /testbed && test -w /testbed && "
+                "test ! -r /opt/mini-swe-agent-venv/pyvenv.cfg && "
+                "test ! -r /installed-agent/setup-env.sh'",
+                log_file=log_file,
+            )
+            if exit_code != 0:
+                raise RuntimeError(
+                    f"Failed to enforce mini-swe-agent filesystem isolation: {output}"
+                )
+        except Exception as e:
+            self.logger.error(f"Failed to prepare mini-swe-agent isolation: {e}")
+            # Isolation is a security boundary. Do not let the runner continue
+            # with root-level tool execution if setup fails.
+            raise
+
+        self.logger.info(
+            "mini-swe-agent tool commands will run as restricted user fbagent"
+        )
         return True
 
-    def failure_hook(self, container, log_file: Path) -> None:
-        """Collect mini-swe-agent output on failures (best-effort)."""
+    def post_run_hook(self, container, log_file) -> bool:
+        """Persist artifacts and accept only an explicit agent submission."""
+        return self._collect_run_artifacts(container, log_file)
+
+    def _collect_run_artifacts(self, container, log_file: Path) -> bool:
+        """Copy run artifacts and validate mini-swe-agent's terminal status."""
         log_dir = Path(log_file).parent
         try:
             self.cm.copy_from_container(
@@ -128,29 +172,67 @@ echo "mini-swe-agent installation complete"
                 "/agent-logs/mini_swe_agent_output.log",
                 log_dir / "mini_swe_agent_output.log",
             )
-        except Exception:
-            pass
+        except Exception as e:
+            self.logger.warning(f"Failed to collect mini-swe-agent output: {e}")
+
+        trajectory_path = log_dir / "traj.json"
         try:
             self.cm.copy_from_container(
                 container,
                 "/root/.config/mini-swe-agent/last_mini_run.traj.json",
-                log_dir / "traj.json",
+                trajectory_path,
             )
-        except Exception:
-            pass
+        except Exception as e:
+            self.logger.error(f"Failed to collect mini-swe-agent trajectory: {e}")
+            return False
+
+        try:
+            trajectory = json.loads(trajectory_path.read_text(encoding="utf-8"))
+            exit_status = trajectory.get("info", {}).get("exit_status")
+        except (OSError, UnicodeError, json.JSONDecodeError, AttributeError) as e:
+            self.logger.error(f"Failed to read mini-swe-agent exit status: {e}")
+            return False
+
+        if not isinstance(exit_status, str) or not exit_status.strip():
+            self.logger.error("mini-swe-agent trajectory has no exit_status")
+            return False
+
+        self.agent_exit_status = exit_status.strip()
+        if self.agent_exit_status != "Submitted":
+            self.logger.warning(
+                f"mini-swe-agent exited without submission: {self.agent_exit_status}"
+            )
+            return False
+
+        self.logger.info("mini-swe-agent exit status: Submitted")
+        return True
+
+    def failure_hook(self, container, log_file: Path) -> None:
+        """Collect mini-swe-agent output on failures (best-effort)."""
+        trajectory_path = Path(log_file).parent / "traj.json"
+        if not trajectory_path.exists():
+            self._collect_run_artifacts(container, log_file)
 
     def get_run_command(self, instruction: str) -> str:
         """Get the command to run mini-swe-agent."""
         escaped_instruction = shlex.quote(instruction)
         model_name = self._get_model_name()
+        cost_limit = self._kwargs.get("cost_limit")
+        cost_limit_arg = ""
+        if cost_limit is not None:
+            cost_limit_arg = f"--cost-limit {shlex.quote(str(cost_limit))} "
         return (
             "set -o pipefail; "
             f'"$MINI_SWE_AGENT_PYTHON" -I - -m {shlex.quote(model_name)} -t {escaped_instruction} -y --exit-immediately '
+            f"{cost_limit_arg}"
+            "--environment-class "
+            "featurebench_mini_swe_environment.RestrictedLocalEnvironment "
             "<<'PY' | tee /agent-logs/mini_swe_agent_output.log\n"
             "import runpy\n"
             "import sys\n"
             "\n"
             "sys.path = [p for p in sys.path if p and not p.startswith('/testbed')]\n"
+            "sys.path.insert(0, '/opt/featurebench-controller')\n"
             "sys.argv = ['minisweagent.run.mini', *sys.argv[1:]]\n"
             "runpy.run_module('minisweagent.run.mini', run_name='__main__')\n"
             "PY"

featurebench-0.2.3/featurebench/infer/environments/__init__.py

@@ -0,0 +1 @@
+"""Runtime environments injected into external inference agents."""