featurebench 0.2.1__tar.gz → 0.2.3__tar.gz

{featurebench-0.2.1 → featurebench-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: featurebench
-Version: 0.2.1
+Version: 0.2.3
 Summary: FeatureBench Pipeline - A test-driven data generation pipeline for building and evaluating feature-level coding benchmarks
 License-File: LICENSE
 Requires-Python: >=3.12
@@ -41,6 +41,8 @@ It provides a unified CLI to run inference, evaluation, and dataset generation.
 
 ## 📰 News
 
+📦 **2026.08.24**: We released **FeatureBench Dataset V1.1**, primarily correcting inconsistencies and omissions in task problem statements.
+
 📊 **2026.05.18**: We added **lite split** evaluation results for frontier models including **GPT-5.5, Claude Opus 4.7, DeepSeek-V4, GLM-5.1, Kimi-2.6, Mimo-V2.5-Pro**, and more to the leaderboard.
 
 🚀 **2026.03.27**: We released the **fast split** containing 100 instances (a subset of full split). These instances require no GPU and are optimized for rapid evaluation. On an Intel Xeon Platinum 8457C with 944GB RAM, the average evaluation time per instance using gold patches is **57.2 seconds**.
@@ -50,7 +52,7 @@ It provides a unified CLI to run inference, evaluation, and dataset generation.
 <!-- LEADERBOARD_START -->
 ## 🏆 Leaderboard
 
-[Full interactive leaderboard](https://libercoders.github.io/FeatureBench/leaderboard/) with tabs, filters, and sorting.
+[Full interactive leaderboard](https://libercoders.github.io/FeatureBench/leaderboard/) with tabs, filters, and sorting (Dataset V1.0).
 
 <details open>
 <summary><b>Lite split results, ranked by %PASSED</b></summary>
@@ -121,6 +123,7 @@ fb pull --mode /path/to/images.txt  # one image name per line
 ```bash
 fb infer \
     --config-path config.toml \
+    --data-version v1.1 \
     --agent mini_swe_agent \
     --model openai/qwen3-coder-480b-a35b-instruct \
     --split fast
@@ -156,4 +159,4 @@ If you found FeatureBench useful, please cite us as:
 
 ## 📧 Contact
 
-If you have any questions, feel free to contact [qixingzhou1125@gmail.com](mailto:qixingzhou1125@gmail.com) or [zjcheng2022@gmail.com](mailto:zjcheng2022@gmail.com).
+If you have any questions, feel free to contact [qixing.zhou@smail.nju.edu.cn](mailto:qixing.zhou@smail.nju.edu.cn) or [zjcheng2022@gmail.com](mailto:zjcheng2022@gmail.com).

{featurebench-0.2.1 → featurebench-0.2.3}/README.md

@@ -17,6 +17,8 @@ It provides a unified CLI to run inference, evaluation, and dataset generation.
 
 ## 📰 News
 
+📦 **2026.08.24**: We released **FeatureBench Dataset V1.1**, primarily correcting inconsistencies and omissions in task problem statements.
+
 📊 **2026.05.18**: We added **lite split** evaluation results for frontier models including **GPT-5.5, Claude Opus 4.7, DeepSeek-V4, GLM-5.1, Kimi-2.6, Mimo-V2.5-Pro**, and more to the leaderboard.
 
 🚀 **2026.03.27**: We released the **fast split** containing 100 instances (a subset of full split). These instances require no GPU and are optimized for rapid evaluation. On an Intel Xeon Platinum 8457C with 944GB RAM, the average evaluation time per instance using gold patches is **57.2 seconds**.
@@ -26,7 +28,7 @@ It provides a unified CLI to run inference, evaluation, and dataset generation.
 <!-- LEADERBOARD_START -->
 ## 🏆 Leaderboard
 
-[Full interactive leaderboard](https://libercoders.github.io/FeatureBench/leaderboard/) with tabs, filters, and sorting.
+[Full interactive leaderboard](https://libercoders.github.io/FeatureBench/leaderboard/) with tabs, filters, and sorting (Dataset V1.0).
 
 <details open>
 <summary><b>Lite split results, ranked by %PASSED</b></summary>
@@ -97,6 +99,7 @@ fb pull --mode /path/to/images.txt  # one image name per line
 ```bash
 fb infer \
     --config-path config.toml \
+    --data-version v1.1 \
     --agent mini_swe_agent \
     --model openai/qwen3-coder-480b-a35b-instruct \
     --split fast
@@ -132,4 +135,4 @@ If you found FeatureBench useful, please cite us as:
 
 ## 📧 Contact
 
-If you have any questions, feel free to contact [qixingzhou1125@gmail.com](mailto:qixingzhou1125@gmail.com) or [zjcheng2022@gmail.com](mailto:zjcheng2022@gmail.com).
+If you have any questions, feel free to contact [qixing.zhou@smail.nju.edu.cn](mailto:qixing.zhou@smail.nju.edu.cn) or [zjcheng2022@gmail.com](mailto:zjcheng2022@gmail.com).

{featurebench-0.2.1 → featurebench-0.2.3}/config_example.toml

@@ -6,6 +6,14 @@ HF_ENDPOINT=""
 HF_TOKEN=""
 GITHUB_TOKEN=""
 
+# ==============================================================================
+# Hugging Face Dataset
+# ==============================================================================
+[dataset]
+# Dataset tag, branch, or commit SHA used by both `fb infer` and `fb eval`.
+# The CLI flag `--data-version` overrides this value.
+revision="v1.1"
+
 # ==============================================================================
 # DataPipeline Configuration
 # ==============================================================================
@@ -44,7 +52,8 @@ base_url = "http://localhost:8080/v1"                # vLLM server URL
 # Optional: Host cache directory for agent downloads; will be mounted to /download inside containers
 # Set to empty to disable cache mount (each container will re-download/install)
 # Use an absolute path here
-download_cache_dir="/path/to/FeatureBench/download_cache"
+# Optional shared install cache. Leave unset for leakage-resistant evaluations.
+# download_cache_dir="/path/to/FeatureBench/download_cache"
 
 # OpenHands Agent Configuration
 [infer_config.openhands]
@@ -109,4 +118,4 @@ MSWEA_BASE_URL = ""
 # Optional: Cost tracking mode (default behavior in FB adapter is ignore_errors)
 MSWEA_COST_TRACKING = ""
 # Optional: Lock mini_swe_agent version (leave empty to use latest)
-MINI_SWE_AGENT_VERSION = ""
+MINI_SWE_AGENT_VERSION = ""

{featurebench-0.2.1 → featurebench-0.2.3}/docs/config.md

@@ -31,13 +31,25 @@ HF_TOKEN = ""
 GITHUB_TOKEN = ""
 ```
 
+### [dataset] (dataset version used by harness / infer)
+
+```toml
+[dataset]
+revision = "v1.1"
+```
+
+- `revision`: Hugging Face dataset tag, branch, or commit SHA used by both
+  `fb infer` and `fb eval`. The default is `v1.1`.
+- `--data-version` overrides this value for a single command.
+
 ---
 
 ## A. Harness
 
 The core idea of harness is: read the `output.jsonl` produced by infer, apply each patch to the corresponding repository image, run tests, and generate evaluation results.
 
-When harness loads the HuggingFace dataset during evaluation, it uses `HF_ENDPOINT` from `[env_vars]`.
+When harness loads the HuggingFace dataset during evaluation, it uses `HF_ENDPOINT`
+from `[env_vars]` and the pinned revision from `[dataset]`.
 
 ---
 
@@ -56,10 +68,11 @@ How infer reads `config.toml`:
 
 ```toml
 [infer]
-download_cache_dir = "/abs/path/to/FeatureBench/download_cache"
+# Optional shared install cache. Leave unset for leakage-resistant evaluations.
+# download_cache_dir = "/abs/path/to/FeatureBench/download_cache"
 ```
 
-- `download_cache_dir`: host-side cache directory that will be mounted into the container at `/download`.
+- `download_cache_dir`: optional host-side cache directory mounted into the container at `/download`. It is disabled by default because the agent can read shared cache contents during inference.
   - Purpose: speed up agent installs/downloads (e.g. npm cache).
   - Empty means no mount (each container downloads on its own; usually slower).
 
@@ -208,4 +221,4 @@ Example (local vLLM):
 [llm.local]
 backend = "vllm"
 base_url = "http://localhost:8080/v1"
-```
+```

{featurebench-0.2.1 → featurebench-0.2.3}/docs/harness_cli_arg.md

@@ -51,6 +51,11 @@ fb eval \
   HuggingFace dataset repo name (e.g., `LiberCoders/FeatureBench`).  
   Default: `LiberCoders/FeatureBench`.
 
+- `--data-version`
+  Hugging Face dataset tag, branch, or commit SHA.
+  Normally inferred from the inference run's `run_metadata.json`.
+  If unavailable, falls back to `[dataset].revision` or `v1.1`.
+
 - `--split`  
   Dataset split name (e.g., `lite`, `full`).  
   Default: `full`.

{featurebench-0.2.1 → featurebench-0.2.3}/docs/infer_cli_arg.md

@@ -6,6 +6,7 @@ This document describes all CLI arguments supported by `fb infer`.
 
 ```bash
 fb infer \
+  --data-version v1.1 \
   --agent gemini_cli \
   --model gemini-3-pro-preview
 ```
@@ -53,6 +54,12 @@ flags can override metadata (see the argument list below).
   HuggingFace dataset repo name (e.g., `LiberCoders/FeatureBench`).  
   Default: `LiberCoders/FeatureBench` in non-resume mode.
 
+- `--data-version`
+  Hugging Face dataset tag, branch, or commit SHA.
+  Overrides `[dataset].revision` from `config.toml`; default: `v1.1`.
+  Saved into `run_metadata.json`; resume mode keeps the recorded dataset version.
+  This is separate from `--version`, which controls the agent software version.
+
 - `--split`  
   Dataset split name (e.g., `lite`, `full`).  
   Default: `full` in non-resume mode. In resume mode, uses metadata.
@@ -153,6 +160,13 @@ flags can override metadata (see the argument list below).
   Default: no override (OpenHands default applies).  
   Resume mode: ignored (uses metadata).
 
+### mini-swe-agent Only
+
+- `--cost-limit`
+  Per-task cost limit in USD, passed to mini-swe-agent.
+  Set to `0` to disable the limit. If omitted, mini-swe-agent's upstream default applies.
+  Resume mode: ignored (uses metadata).
+
 ## Section 4: Output Directory Structure
 
 ```
@@ -166,4 +180,4 @@ runs/{timestamp}/
             ├── infer.log         # Agent execution log
             ├── run.log           # Runtime log
             └── patch.diff        # Generated patch
-```
+```

{featurebench-0.2.1 → featurebench-0.2.3}/featurebench/cli.py

@@ -31,7 +31,7 @@ def _print_help(stream=None) -> None:
         "  data   Run data pipeline (supports featurebench-pipeline args)\n"
         "\n"
         "Examples:\n"
-        "  fb infer --agent mini_swe_agent --model openai/gpt-4o --split fast\n"
+        "  fb infer --data-version v1.1 --agent mini_swe_agent --model openai/gpt-4o --split fast\n"
         "  fb eval --predictions-path runs/xxx/output.jsonl --split fast\n"
         "  fb pull --mode fast\n"
         "  fb data --config-path constants/python_new.py --output-dir runs/data\n"
@@ -128,4 +128,4 @@ def main(argv: Sequence[str] | None = None) -> int:
 
 
 if __name__ == "__main__":
-    raise SystemExit(main())
+    raise SystemExit(main())

featurebench-0.2.3/featurebench/environment_fixes/__init__.py

@@ -0,0 +1,13 @@
+"""Repository-specific fixes for leaked or duplicated task-environment artifacts.
+
+Add a fix by implementing :class:`EnvironmentFix`, decorating it with
+``register_environment_fix``, and importing its module below. Each plugin is
+matched by repository and is shared by inference and Oracle evaluation.
+"""
+
+from featurebench.environment_fixes.registry import apply_environment_fixes
+
+# Import built-in plugins so they register themselves.
+from featurebench.environment_fixes import mlflow as _mlflow  # noqa: F401, E402
+
+__all__ = ["apply_environment_fixes"]

featurebench-0.2.3/featurebench/environment_fixes/base.py

@@ -0,0 +1,24 @@
+"""Interfaces for repository-specific task-environment fixes."""
+
+from abc import ABC, abstractmethod
+from collections.abc import Callable
+import logging
+
+
+CommandRunner = Callable[[str], tuple[int, str | bytes]]
+
+
+class EnvironmentFix(ABC):
+    """A targeted cleanup applied before the task repository becomes the Git baseline."""
+
+    name: str
+    repositories: frozenset[str]
+
+    def applies_to(self, repository: str) -> bool:
+        """Return whether this fix applies to a canonical ``owner/repo`` name."""
+        normalized = repository.strip().casefold()
+        return normalized in {repo.casefold() for repo in self.repositories}
+
+    @abstractmethod
+    def apply(self, run_command: CommandRunner, logger: logging.Logger) -> bool:
+        """Apply and verify the fix, returning ``False`` on failure."""

featurebench-0.2.3/featurebench/environment_fixes/mlflow.py

@@ -0,0 +1,41 @@
+"""MLflow-specific task-environment cleanup."""
+
+import logging
+
+from featurebench.environment_fixes.base import CommandRunner, EnvironmentFix
+from featurebench.environment_fixes.registry import register_environment_fix
+
+
+@register_environment_fix
+class MlflowParallelSourcesFix(EnvironmentFix):
+    """Remove unmasked MLflow package copies shipped in the monorepo's ``libs`` tree."""
+
+    name = "mlflow_parallel_sources"
+    repositories = frozenset({"mlflow/mlflow"})
+    paths = (
+        "/testbed/libs/skinny/mlflow",
+        "/testbed/libs/tracing/mlflow",
+    )
+
+    def apply(self, run_command: CommandRunner, logger: logging.Logger) -> bool:
+        paths = " ".join(self.paths)
+        command = f"""
+        paths=({paths})
+        for path in "${{paths[@]}}"; do
+            if [ -e "$path" ] || [ -L "$path" ]; then
+                rm -rf -- "$path" || exit 1
+                echo "Removed MLflow parallel source: $path"
+            fi
+            if [ -e "$path" ] || [ -L "$path" ]; then
+                echo "MLflow parallel source still exists: $path" >&2
+                exit 1
+            fi
+        done
+        """
+        exit_code, output = run_command(command)
+        if exit_code != 0:
+            if isinstance(output, bytes):
+                output = output.decode("utf-8", errors="replace")
+            logger.error("Failed to remove MLflow parallel sources: %s", output)
+            return False
+        return True

featurebench-0.2.3/featurebench/environment_fixes/registry.py

@@ -0,0 +1,43 @@
+"""Registry and execution helpers for task-environment fixes."""
+
+import logging
+from typing import TypeVar
+
+from featurebench.environment_fixes.base import CommandRunner, EnvironmentFix
+
+
+_FIXES: list[EnvironmentFix] = []
+_EnvironmentFixType = TypeVar("_EnvironmentFixType", bound=type[EnvironmentFix])
+
+
+def register_environment_fix(fix_type: _EnvironmentFixType) -> _EnvironmentFixType:
+    """Register an environment-fix class at import time."""
+    fix = fix_type()
+    if any(existing.name == fix.name for existing in _FIXES):
+        raise ValueError(f"Duplicate environment fix name: {fix.name}")
+    _FIXES.append(fix)
+    return fix_type
+
+
+def apply_environment_fixes(
+    repository: str,
+    run_command: CommandRunner,
+    logger: logging.Logger,
+) -> bool:
+    """Apply every registered fix matching ``repository`` in registration order."""
+    matching_fixes = [fix for fix in _FIXES if fix.applies_to(repository)]
+    if not matching_fixes:
+        return True
+
+    for fix in matching_fixes:
+        logger.info("Applying environment fix %s for %s", fix.name, repository)
+        try:
+            if not fix.apply(run_command, logger):
+                logger.error("Environment fix %s failed for %s", fix.name, repository)
+                return False
+        except Exception:
+            logger.exception("Environment fix %s raised for %s", fix.name, repository)
+            return False
+        logger.info("Environment fix %s completed for %s", fix.name, repository)
+
+    return True

{featurebench-0.2.1 → featurebench-0.2.3}/featurebench/harness/run_evaluation.py

@@ -74,7 +74,13 @@ from featurebench.harness.utils import (
     parse_repo_settings,
 )
 
-from featurebench.infer.config import InferConfigLoader
+from featurebench.infer.config import (
+    DEFAULT_DATASET_REVISION,
+    InferConfigLoader,
+    configure_hf_access,
+    resolve_hf_dataset_revision,
+    verify_loaded_dataset_revision,
+)
 from featurebench.infer.gpu_scheduler import GpuLease, GpuScheduler, detect_host_gpu_ids, parse_gpu_id_list
 
 # Global console for rich output
@@ -747,6 +753,7 @@ def load_dataset_from_hf(
     split: str,
     dataset: str,
     config_path: str | None = None,
+    dataset_revision: str | None = None,
 ) -> pd.DataFrame:
     """
     Load dataset from HuggingFace using config.toml settings.
@@ -766,21 +773,39 @@ def load_dataset_from_hf(
         resolved_path = Path(config_path).expanduser() if config_path else None
         config_loader = InferConfigLoader(resolved_path)
         env_vars = config_loader.env_vars
-        hf_token = env_vars.get("HF_TOKEN")
+        hf_token, hf_endpoint = configure_hf_access(env_vars)
+        configured_revision = config_loader.get_dataset_revision()
     except FileNotFoundError:
         console.print("[yellow]Warning: config.toml not found, using defaults[/]")
-        hf_token = os.environ.get('HF_TOKEN', None)
+        hf_token, hf_endpoint = configure_hf_access({})
+        configured_revision = DEFAULT_DATASET_REVISION
+
+    requested_revision = str(dataset_revision or "").strip()
+    effective_revision = requested_revision or configured_revision
 
     console.print(f"[bold blue]Loading dataset from HuggingFace...[/]")
     console.print(f"[dim]Repository: {dataset}[/]")
     console.print(f"[dim]Split: {split}[/]")
+    console.print(f"[dim]Version: {effective_revision}[/]")
 
     if hf_token:
         console.print("[dim]Using HuggingFace token from config[/]")
 
     try:
+        resolved_revision = resolve_hf_dataset_revision(
+            dataset,
+            effective_revision,
+            hf_token,
+            hf_endpoint,
+        )
         # Load the specified split
-        hf_dataset = load_dataset(dataset, split=split, token=hf_token)
+        hf_dataset = load_dataset(
+            dataset,
+            split=split,
+            token=hf_token,
+            revision=resolved_revision,
+        )
+        verify_loaded_dataset_revision(hf_dataset, resolved_revision)
         df = pd.DataFrame(hf_dataset)
 
         # Determine level from instance_id's last segment (e.g., "xxx.lv1" -> level 1)
@@ -809,6 +834,28 @@ def load_dataset_from_hf(
         raise
 
 
+def resolve_eval_dataset_revision(
+    cli_revision: str | None,
+    run_metadata: dict | None,
+) -> str | None:
+    """Resolve the dataset revision for evaluation.
+
+    An explicit CLI value wins. Otherwise, evaluation follows the revision
+    recorded by inference. Runs created before revision tracking belong to the
+    original v1.0 dataset. When no run metadata is available, the dataset
+    loader falls back to the configured/default revision.
+    """
+    requested_revision = str(cli_revision or "").strip()
+    if requested_revision:
+        return requested_revision
+
+    if run_metadata:
+        recorded_revision = str(run_metadata.get("dataset_revision") or "").strip()
+        return recorded_revision or "v1.0"
+
+    return None
+
+
 def parse_args() -> argparse.Namespace:
     """Parse command line arguments."""
     parser = argparse.ArgumentParser(
@@ -874,6 +921,17 @@ def parse_args() -> argparse.Namespace:
         help="HuggingFace dataset repo name (e.g., 'LiberCoders/FeatureBench')",
     )
 
+    parser.add_argument(
+        "--data-version",
+        dest="dataset_version",
+        type=str,
+        default=None,
+        help=(
+            "Hugging Face dataset version (tag, branch, or commit SHA). "
+            "Default: infer from run_metadata.json, then [dataset].revision, then v1.1"
+        ),
+    )
+
     parser.add_argument(
         "--split",
         type=str,
@@ -956,15 +1014,23 @@ def main():
     # we keep FAIL_TO_PASS test files visible during Level 1 patch application to avoid
     # "patch touches deleted test file" apply failures.
     white_enabled = False
+    run_metadata = None
     try:
         metadata_path = output_dir / "run_metadata.json"
-        if metadata_path.exists():
+        if not use_gold_patch and metadata_path.exists():
             with open(metadata_path, "r", encoding="utf-8") as f:
                 meta = json.load(f)
-            if isinstance(meta, dict) and meta.get("white_box") is True:
-                white_enabled = True
+            if isinstance(meta, dict):
+                run_metadata = meta
+                white_enabled = meta.get("white_box") is True
     except Exception:
         white_enabled = False
+        run_metadata = None
+
+    dataset_revision = resolve_eval_dataset_revision(
+        args.dataset_version,
+        run_metadata,
+    )
 
     # Print startup banner
     console.print()
@@ -996,7 +1062,13 @@ def main():
     console.print()
 
     # Load dataset
-    dataset = load_dataset_from_hf(console, args.split, args.dataset, args.config_path)
+    dataset = load_dataset_from_hf(
+        console,
+        args.split,
+        args.dataset,
+        args.config_path,
+        dataset_revision,
+    )
 
     # Load predictions
     console.print(f"[bold blue]Loading predictions...[/]")
@@ -1221,4 +1293,4 @@ def main():
 
 
 if __name__ == "__main__":
-    raise SystemExit(main())
+    raise SystemExit(main())

{featurebench-0.2.1 → featurebench-0.2.3}/featurebench/harness/runtime.py

@@ -13,6 +13,7 @@ from typing import Any
 import pandas as pd
 from docker.models.containers import Container
 
+from featurebench.environment_fixes import apply_environment_fixes
 from featurebench.harness.constants import (
     APPLY_PATCH_FAIL,
     APPLY_PATCH_PASS,
@@ -55,9 +56,11 @@ def run_instance_level1(
     1. Activate conda environment
     2. Restore project from /root/my_repo/
     3. Apply patch (for masking) and directly delete F2P test files
-    4. Reinitialize git
-    5. Apply agent's patch
-    6. Delete any generated F2P files, restore F2P test, and run tests
+    4. Apply repository-specific environment fixes
+    5. Reinitialize git
+    6. Apply agent's patch
+    7. Delete any generated F2P files and restore the F2P test
+    8. Run tests
 
     Args:
         instance: Instance data (from HuggingFace)
@@ -177,8 +180,20 @@ def run_instance_level1(
             else:
                 logger.warning("No FAIL_TO_PASS tests found in instance, skipping test file deletion")
 
-        # Step 3: Reinitialize git repository
-        logger.info("Step 3: Reinitializing git repository")
+        # Step 3: Apply the same repository-specific cleanup used during
+        # inference. Running it before Git initialization keeps the cleanup out
+        # of the candidate patch while making Oracle validation representative.
+        logger.info("Step 3: Applying repository-specific environment fixes")
+        if not apply_environment_fixes(
+            str(repo_name),
+            lambda command: exec_run_with_timeout(container, command, timeout=120),
+            logger,
+        ):
+            results["error"] = "Repository-specific environment fix failed"
+            return results
+
+        # Step 4: Reinitialize git repository
+        logger.info("Step 4: Reinitializing git repository")
         git_cmds = [
             "cd /testbed && rm -rf .git",
             "cd /testbed && git init",
@@ -194,8 +209,8 @@ def run_instance_level1(
                 logger.warning(f"Git command failed: {cmd}")
                 logger.warning(f"Output: {output.decode(UTF8, errors='replace')}")
 
-        # Step 4: Apply agent's patch
-        logger.info("Step 4: Applying agent patch")
+        # Step 5: Apply agent's patch
+        logger.info("Step 5: Applying agent patch")
         patch_content = pred[KEY_PREDICTION]
 
         if not patch_content or patch_content.strip() == "":
@@ -231,8 +246,8 @@ def run_instance_level1(
         logger.info(f"{APPLY_PATCH_PASS}")
         results["patch_applied"] = True
 
-        # Step 5: Prepare for testing - delete generated F2P files and restore F2P test
-        logger.info("Step 5: Preparing for testing - cleaning up and restoring F2P test")
+        # Step 6: Prepare for testing - delete generated F2P files and restore F2P test
+        logger.info("Step 6: Preparing for testing - cleaning up and restoring F2P test")
 
         fail_to_pass = instance.get('FAIL_TO_PASS', [])
         if not fail_to_pass:
@@ -268,8 +283,8 @@ def run_instance_level1(
         else:
             logger.warning("No test_patch available to restore F2P test file")
 
-        # Step 6: Run tests
-        logger.info("Step 6: Running tests")
+        # Step 7: Run tests
+        logger.info("Step 7: Running tests")
 
         # Get test configuration from repo_settings
         repo_settings = parse_repo_settings(instance)
@@ -589,4 +604,3 @@ def run_instance_level2(
         logger.error(traceback.format_exc())
         results["error"] = str(e)
         return results
-

{featurebench-0.2.1 → featurebench-0.2.3}/featurebench/infer/agents/base.py

@@ -41,6 +41,10 @@ class BaseAgent(ABC):
         self.env_vars = env_vars or {}
         self.logger = logger or logging.getLogger(__name__)
         self._kwargs = kwargs
+        # Agent adapters may populate this with their native terminal status.
+        # It deliberately remains optional because not every supported agent
+        # exposes a structured status artifact.
+        self.agent_exit_status: Optional[str] = None
     
     @property
     @abstractmethod
@@ -354,6 +358,7 @@ class BaseAgent(ABC):
         Returns:
             True if agent completed successfully
         """
+        self.agent_exit_status = None
         self.logger.info(f"Running {self.name} agent...")
         
         with open(log_file, "a", encoding="utf-8") as f:
@@ -385,16 +390,23 @@ class BaseAgent(ABC):
                 log_file=log_file,
                 timeout=timeout
             )
-            
+
             success_run = exit_code == 0
+            timed_out = not success_run and self._infer_log_has_timeout_marker(log_file)
             success_post_run = self.post_run_hook(container, log_file)
 
+            # A host-side timeout may kill the agent before it can persist a
+            # native terminal status. Preserve the actual reason for stopping
+            # instead of reporting an unknown/generic failure.
+            if timed_out:
+                self.agent_exit_status = "TimeExceeded"
+
             if success_run and success_post_run:
                 self.logger.info(f"{self.name} agent completed successfully")
                 return True
 
             if not success_run:
-                if self._force_timeout_enabled() and self._infer_log_has_timeout_marker(log_file):
+                if self._force_timeout_enabled() and timed_out:
                     post_run_state = "passed" if success_post_run else "failed"
                     self.logger.warning(
                         f"{self.name} run hit timeout marker (post-run hook {post_run_state}); "