featurebench 0.2.0__tar.gz → 0.2.2__tar.gz

{featurebench-0.2.0 → featurebench-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: featurebench
-Version: 0.2.0
+Version: 0.2.2
 Summary: FeatureBench Pipeline - A test-driven data generation pipeline for building and evaluating feature-level coding benchmarks
 License-File: LICENSE
 Requires-Python: >=3.12
@@ -41,6 +41,8 @@ It provides a unified CLI to run inference, evaluation, and dataset generation.
 
 ## 📰 News
 
+📦 **2026.08.24**: We released **FeatureBench Dataset V1.1**, primarily correcting inconsistencies and omissions in task problem statements.
+
 📊 **2026.05.18**: We added **lite split** evaluation results for frontier models including **GPT-5.5, Claude Opus 4.7, DeepSeek-V4, GLM-5.1, Kimi-2.6, Mimo-V2.5-Pro**, and more to the leaderboard.
 
 🚀 **2026.03.27**: We released the **fast split** containing 100 instances (a subset of full split). These instances require no GPU and are optimized for rapid evaluation. On an Intel Xeon Platinum 8457C with 944GB RAM, the average evaluation time per instance using gold patches is **57.2 seconds**.
@@ -75,7 +77,8 @@ It provides a unified CLI to run inference, evaluation, and dataset generation.
 | 16 | GLM 4.7 | Mini-SWE-Agent | 41.2 | 6.7 |
 | 17 | Qwen3-Coder-480B-A35B-Instruct | OpenHands | 38.3 | 6.7 |
 | 18 | DeepSeek V3.2 | OpenHands | 35.9 | 6.7 |
-| 19 | Qwen3-Coder-30B-A3B-Instruct | OpenHands | 23 | 3.3 |
+| 19 | Qwen3.5-27B | OpenHands | 34.8 | 10.0 |
+| 20 | Qwen3-Coder-30B-A3B-Instruct | OpenHands | 23 | 3.3 |
 
 </details>
 <!-- LEADERBOARD_END -->
@@ -120,6 +123,7 @@ fb pull --mode /path/to/images.txt  # one image name per line
 ```bash
 fb infer \
     --config-path config.toml \
+    --data-version v1.1 \
     --agent mini_swe_agent \
     --model openai/qwen3-coder-480b-a35b-instruct \
     --split fast
@@ -155,4 +159,4 @@ If you found FeatureBench useful, please cite us as:
 
 ## 📧 Contact
 
-If you have any questions, feel free to contact [qixingzhou1125@gmail.com](mailto:qixingzhou1125@gmail.com) or [zjcheng2022@gmail.com](mailto:zjcheng2022@gmail.com).
+If you have any questions, feel free to contact [zhouqixing@stu.cqu.edu.cn](mailto:zhouqixing@stu.cqu.edu.cn) or [zjcheng2022@gmail.com](mailto:zjcheng2022@gmail.com).

{featurebench-0.2.0 → featurebench-0.2.2}/README.md

@@ -17,6 +17,8 @@ It provides a unified CLI to run inference, evaluation, and dataset generation.
 
 ## 📰 News
 
+📦 **2026.08.24**: We released **FeatureBench Dataset V1.1**, primarily correcting inconsistencies and omissions in task problem statements.
+
 📊 **2026.05.18**: We added **lite split** evaluation results for frontier models including **GPT-5.5, Claude Opus 4.7, DeepSeek-V4, GLM-5.1, Kimi-2.6, Mimo-V2.5-Pro**, and more to the leaderboard.
 
 🚀 **2026.03.27**: We released the **fast split** containing 100 instances (a subset of full split). These instances require no GPU and are optimized for rapid evaluation. On an Intel Xeon Platinum 8457C with 944GB RAM, the average evaluation time per instance using gold patches is **57.2 seconds**.
@@ -51,7 +53,8 @@ It provides a unified CLI to run inference, evaluation, and dataset generation.
 | 16 | GLM 4.7 | Mini-SWE-Agent | 41.2 | 6.7 |
 | 17 | Qwen3-Coder-480B-A35B-Instruct | OpenHands | 38.3 | 6.7 |
 | 18 | DeepSeek V3.2 | OpenHands | 35.9 | 6.7 |
-| 19 | Qwen3-Coder-30B-A3B-Instruct | OpenHands | 23 | 3.3 |
+| 19 | Qwen3.5-27B | OpenHands | 34.8 | 10.0 |
+| 20 | Qwen3-Coder-30B-A3B-Instruct | OpenHands | 23 | 3.3 |
 
 </details>
 <!-- LEADERBOARD_END -->
@@ -96,6 +99,7 @@ fb pull --mode /path/to/images.txt  # one image name per line
 ```bash
 fb infer \
     --config-path config.toml \
+    --data-version v1.1 \
     --agent mini_swe_agent \
     --model openai/qwen3-coder-480b-a35b-instruct \
     --split fast
@@ -131,4 +135,4 @@ If you found FeatureBench useful, please cite us as:
 
 ## 📧 Contact
 
-If you have any questions, feel free to contact [qixingzhou1125@gmail.com](mailto:qixingzhou1125@gmail.com) or [zjcheng2022@gmail.com](mailto:zjcheng2022@gmail.com).
+If you have any questions, feel free to contact [zhouqixing@stu.cqu.edu.cn](mailto:zhouqixing@stu.cqu.edu.cn) or [zjcheng2022@gmail.com](mailto:zjcheng2022@gmail.com).

{featurebench-0.2.0 → featurebench-0.2.2}/config_example.toml

@@ -6,6 +6,14 @@ HF_ENDPOINT=""
 HF_TOKEN=""
 GITHUB_TOKEN=""
 
+# ==============================================================================
+# Hugging Face Dataset
+# ==============================================================================
+[dataset]
+# Dataset tag, branch, or commit SHA used by both `fb infer` and `fb eval`.
+# The CLI flag `--data-version` overrides this value.
+revision="v1.1"
+
 # ==============================================================================
 # DataPipeline Configuration
 # ==============================================================================
@@ -53,13 +61,15 @@ LLM_API_KEY = ""
 # Optional: LLM API Base URL
 LLM_BASE_URL = ""
 # Optional: Lock OpenHands version (leave empty to use latest)
-OPENHANDS_VERSION = "0.62.0"
+OPENHANDS_VERSION = ""
 # Optional: Reasoning effort for OpenAI o-series models
 LLM_REASONING_EFFORT = ""
 # Optional: Force OpenHands native tool calling ("true" or "false"; empty uses OpenHands default)
 LLM_NATIVE_TOOL_CALLING = ""
 # Optional: Send prior assistant reasoning_content back to OpenHands requests
 LLM_SEND_REASONING_CONTENT = ""
+# Optional: JSON object passed to LiteLLM extra_body, e.g. {"enable_thinking": true}
+LLM_LITELLM_EXTRA_BODY = ""
 # Azure API Version (required for Azure models only)
 LLM_API_VERSION = ""
 # Optional: OpenHands agent max iterations (step limit). Upstream default is 500.
@@ -107,4 +117,4 @@ MSWEA_BASE_URL = ""
 # Optional: Cost tracking mode (default behavior in FB adapter is ignore_errors)
 MSWEA_COST_TRACKING = ""
 # Optional: Lock mini_swe_agent version (leave empty to use latest)
-MINI_SWE_AGENT_VERSION = ""
+MINI_SWE_AGENT_VERSION = ""

{featurebench-0.2.0 → featurebench-0.2.2}/docs/config.md

@@ -31,13 +31,25 @@ HF_TOKEN = ""
 GITHUB_TOKEN = ""
 ```
 
+### [dataset] (dataset version used by harness / infer)
+
+```toml
+[dataset]
+revision = "v1.1"
+```
+
+- `revision`: Hugging Face dataset tag, branch, or commit SHA used by both
+  `fb infer` and `fb eval`. The default is `v1.1`.
+- `--data-version` overrides this value for a single command.
+
 ---
 
 ## A. Harness
 
 The core idea of harness is: read the `output.jsonl` produced by infer, apply each patch to the corresponding repository image, run tests, and generate evaluation results.
 
-When harness loads the HuggingFace dataset during evaluation, it uses `HF_ENDPOINT` from `[env_vars]`.
+When harness loads the HuggingFace dataset during evaluation, it uses `HF_ENDPOINT`
+from `[env_vars]` and the pinned revision from `[dataset]`.
 
 ---
 
@@ -77,6 +89,7 @@ INFER_LOG_RENDER_MODE = "compact" # Optional: compact|full for infer.log renderi
 LLM_REASONING_EFFORT = ""     # Optional: Reasoning effort for OpenAI o-series models
 LLM_NATIVE_TOOL_CALLING = ""  # Optional: true/false to force OpenHands native tool calling; empty uses OpenHands default
 LLM_SEND_REASONING_CONTENT = "" # Optional: true to send prior assistant reasoning_content in history
+LLM_LITELLM_EXTRA_BODY = "" # Optional: JSON object passed to LiteLLM extra_body, e.g. {"enable_thinking": true}
 OPENHANDS_MAX_ITERATIONS = "" # Optional: OpenHands agent max iterations (step limit). Upstream default is 500.
 
 ```
@@ -207,4 +220,4 @@ Example (local vLLM):
 [llm.local]
 backend = "vllm"
 base_url = "http://localhost:8080/v1"
-```
+```

{featurebench-0.2.0 → featurebench-0.2.2}/docs/harness_cli_arg.md

@@ -51,6 +51,11 @@ fb eval \
   HuggingFace dataset repo name (e.g., `LiberCoders/FeatureBench`).  
   Default: `LiberCoders/FeatureBench`.
 
+- `--data-version`
+  Hugging Face dataset tag, branch, or commit SHA.
+  Normally inferred from the inference run's `run_metadata.json`.
+  If unavailable, falls back to `[dataset].revision` or `v1.1`.
+
 - `--split`  
   Dataset split name (e.g., `lite`, `full`).  
   Default: `full`.

{featurebench-0.2.0 → featurebench-0.2.2}/docs/infer_cli_arg.md

@@ -6,6 +6,7 @@ This document describes all CLI arguments supported by `fb infer`.
 
 ```bash
 fb infer \
+  --data-version v1.1 \
   --agent gemini_cli \
   --model gemini-3-pro-preview
 ```
@@ -53,6 +54,12 @@ flags can override metadata (see the argument list below).
   HuggingFace dataset repo name (e.g., `LiberCoders/FeatureBench`).  
   Default: `LiberCoders/FeatureBench` in non-resume mode.
 
+- `--data-version`
+  Hugging Face dataset tag, branch, or commit SHA.
+  Overrides `[dataset].revision` from `config.toml`; default: `v1.1`.
+  Saved into `run_metadata.json`; resume mode keeps the recorded dataset version.
+  This is separate from `--version`, which controls the agent software version.
+
 - `--split`  
   Dataset split name (e.g., `lite`, `full`).  
   Default: `full` in non-resume mode. In resume mode, uses metadata.
@@ -135,9 +142,17 @@ flags can override metadata (see the argument list below).
   Force native tool calling off (`LLM_NATIVE_TOOL_CALLING=false`).  
   Resume mode: ignored (uses metadata).
 
-- `--send-reasoning-content`  
-  Send prior assistant `reasoning_content` back to the model in subsequent OpenHands requests.  
-  Useful for thinking models whose chat template supports reasoning history.  
+- `--send-reasoning-content`
+  Send prior assistant reasoning back to the model in subsequent OpenHands requests using both `reasoning_content` and `reasoning` fields.
+  Useful for thinking models whose chat template supports reasoning history.
+  Resume mode: ignored (uses metadata).
+
+- `--litellm-extra-body`  
+  Pass a JSON object to OpenHands `LLM.litellm_extra_body`, for example `--litellm-extra-body '{"enable_thinking": true}'`.  
+  Resume mode: ignored (uses metadata).
+
+- `--session-cache`
+  Enable backend KV-cache session reuse for OpenHands by sending a unique `X-Session-Id` header for each task attempt and releasing it after the run with `DELETE /session_release?session_id=...`.
   Resume mode: ignored (uses metadata).
 
 - `--max-iters`  
@@ -158,4 +173,4 @@ runs/{timestamp}/
             ├── infer.log         # Agent execution log
             ├── run.log           # Runtime log
             └── patch.diff        # Generated patch
-```
+```

{featurebench-0.2.0 → featurebench-0.2.2}/featurebench/cli.py

@@ -31,7 +31,7 @@ def _print_help(stream=None) -> None:
         "  data   Run data pipeline (supports featurebench-pipeline args)\n"
         "\n"
         "Examples:\n"
-        "  fb infer --agent mini_swe_agent --model openai/gpt-4o --split fast\n"
+        "  fb infer --data-version v1.1 --agent mini_swe_agent --model openai/gpt-4o --split fast\n"
         "  fb eval --predictions-path runs/xxx/output.jsonl --split fast\n"
         "  fb pull --mode fast\n"
         "  fb data --config-path constants/python_new.py --output-dir runs/data\n"
@@ -56,6 +56,8 @@ def _run_with_patched_argv(
         if result is None:
             return 0
         return int(result)
+    except KeyboardInterrupt:
+        return 130
     except SystemExit as exc:
         code = exc.code
         if code is None:

{featurebench-0.2.0 → featurebench-0.2.2}/featurebench/docker/image_manager.py

@@ -1,4 +1,5 @@
 import logging
+import atexit
 import subprocess
 import tempfile
 import os
@@ -6,6 +7,8 @@ import hashlib
 import platform
 import importlib.util
 import shutil
+import signal
+import threading
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from typing import Dict, Optional, List, Tuple
 from pathlib import Path
@@ -24,6 +27,11 @@ from featurebench.utils.logger import (
 from featurebench.utils.utils import select_candidate_pool_and_allocate_gpu, release_gpu
 
 
+FEATUREBENCH_RUN_LABEL = "featurebench.run"
+FEATUREBENCH_KIND_LABEL = "featurebench.kind"
+FEATUREBENCH_TASK_LABEL = "featurebench.task"
+
+
 class ImageManager:
     """Docker image manager."""
     
@@ -52,6 +60,159 @@ class ImageManager:
         
         # Track GPUs per container: {container_id: [gpu_id, ...]}
         self._container_gpu_map: Dict[str, List[int]] = {}
+
+        # Track containers created by fb data so Ctrl+C/process-exit cleanup can
+        # remove them even if worker threads do not reach their own finally block.
+        self._cleanup_run_id = f"data-{int(time.time() * 1000)}-{os.getpid()}"
+        self._active_container_ids_lock = threading.RLock()
+        self._active_container_ids: set[str] = set()
+        self._cleanup_lock = threading.RLock()
+        self._cleanup_in_progress = False
+        self._cleanup_interrupt_notice_printed = False
+        self._previous_sigint = None
+        self._signal_cleanup_installed = False
+        if threading.current_thread() is threading.main_thread():
+            self._previous_sigint = signal.getsignal(signal.SIGINT)
+            signal.signal(signal.SIGINT, self._handle_interrupt)
+            self._signal_cleanup_installed = True
+        self._atexit_cleanup = self._cleanup_active_containers_at_exit
+        atexit.register(self._atexit_cleanup)
+
+    def _container_labels(self, specs_name: str, purpose: str = "data") -> Dict[str, str]:
+        return {
+            FEATUREBENCH_RUN_LABEL: self._cleanup_run_id,
+            FEATUREBENCH_KIND_LABEL: "data",
+            FEATUREBENCH_TASK_LABEL: str(specs_name),
+            "featurebench.purpose": purpose,
+        }
+
+    def _add_container_label_args(self, docker_command: List[str], specs_name: str) -> None:
+        for key, value in self._container_labels(specs_name).items():
+            docker_command.extend(["--label", f"{key}={value}"])
+
+    def _register_container_id(self, container_id: str) -> None:
+        if not container_id:
+            return
+        with self._active_container_ids_lock:
+            self._active_container_ids.add(container_id)
+
+    def _unregister_container_id(self, container_id: str) -> None:
+        if not container_id:
+            return
+        with self._active_container_ids_lock:
+            self._active_container_ids.discard(container_id)
+
+    def _ignore_interrupt_during_cleanup(self, signum, frame) -> None:
+        if not self._cleanup_interrupt_notice_printed:
+            self._cleanup_interrupt_notice_printed = True
+            try:
+                self.logger.warning("Cleanup already in progress; ignoring additional Ctrl+C.")
+            except Exception:
+                pass
+
+    def _handle_interrupt(self, signum, frame) -> None:
+        try:
+            self.logger.warning("Interrupted; cleaning FeatureBench data containers...")
+        except Exception:
+            pass
+        self.cleanup_active_containers("keyboard interrupt")
+
+        previous = self._previous_sigint
+        if callable(previous):
+            previous(signum, frame)
+        elif previous == signal.SIG_DFL:
+            raise KeyboardInterrupt
+
+    def _release_container_gpu(self, container_id: str) -> None:
+        gpu_ids = self._container_gpu_map.pop(container_id, None)
+        if gpu_ids:
+            release_gpu(gpu_ids, self.logger)
+            self.logger.debug(f"Container {container_id[:12]} released GPUs {gpu_ids}")
+
+    def _remove_container_id_best_effort(self, container_id: str) -> bool:
+        if not container_id:
+            return False
+        try:
+            self._release_container_gpu(container_id)
+            subprocess.run(["docker", "kill", container_id], capture_output=True)
+            subprocess.run(["docker", "rm", "-f", container_id], capture_output=True)
+            return True
+        finally:
+            self._unregister_container_id(container_id)
+
+    def _cleanup_labeled_containers(self) -> int:
+        try:
+            result = subprocess.run(
+                [
+                    "docker",
+                    "ps",
+                    "-aq",
+                    "--filter",
+                    f"label={FEATUREBENCH_RUN_LABEL}={self._cleanup_run_id}",
+                    "--filter",
+                    f"label={FEATUREBENCH_KIND_LABEL}=data",
+                ],
+                capture_output=True,
+                text=True,
+                check=False,
+            )
+        except Exception as exc:
+            self.logger.warning(f"Failed to scan FeatureBench data containers by label: {exc}")
+            return 0
+
+        if result.returncode != 0:
+            stderr = (result.stderr or "").strip()
+            if stderr:
+                self.logger.warning(f"Failed to scan FeatureBench data containers by label: {stderr}")
+            return 0
+
+        removed = 0
+        for container_id in result.stdout.splitlines():
+            if self._remove_container_id_best_effort(container_id.strip()):
+                removed += 1
+        return removed
+
+    def cleanup_active_containers(self, reason: str) -> int:
+        with self._cleanup_lock:
+            if self._cleanup_in_progress:
+                return 0
+            self._cleanup_in_progress = True
+
+        previous_sigint = None
+        signal_replaced = False
+        try:
+            if threading.current_thread() is threading.main_thread():
+                previous_sigint = signal.getsignal(signal.SIGINT)
+                signal.signal(signal.SIGINT, self._ignore_interrupt_during_cleanup)
+                signal_replaced = True
+
+            with self._active_container_ids_lock:
+                container_ids = list(self._active_container_ids)
+
+            removed = 0
+            if container_ids:
+                self.logger.warning(
+                    f"Cleaning {len(container_ids)} active FeatureBench data container(s) after {reason}..."
+                )
+            for container_id in container_ids:
+                if self._remove_container_id_best_effort(container_id):
+                    removed += 1
+
+            removed += self._cleanup_labeled_containers()
+            if removed:
+                self.logger.warning(f"Removed {removed} FeatureBench data container(s).")
+            return removed
+        finally:
+            if signal_replaced:
+                try:
+                    signal.signal(signal.SIGINT, previous_sigint)
+                except Exception:
+                    pass
+            with self._cleanup_lock:
+                self._cleanup_in_progress = False
+
+    def _cleanup_active_containers_at_exit(self) -> None:
+        self.cleanup_active_containers("process exit")
     
     def prepare_images(self, repo_manager: RepoManager) -> None:
         """
@@ -664,6 +825,7 @@ RUN echo "source /opt/miniconda3/etc/profile.d/conda.sh && conda activate testbe
         
         # Add runtime config to docker_command and compute env_setup + GPU IDs
         env_setup_cmd, selected_gpu_ids = self._add_docker_runtime_config(docker_command, specs)
+        self._add_container_label_args(docker_command, specs_name)
         
         # Add image
         docker_command.append(instance_image)
@@ -695,6 +857,7 @@ RUN echo "source /opt/miniconda3/etc/profile.d/conda.sh && conda activate testbe
         )
         
         container_id = result.stdout.strip()
+        self._register_container_id(container_id)
         
         # Record GPUs used by container (if any)
         if selected_gpu_ids:
@@ -1097,17 +1260,16 @@ RUN echo "source /opt/miniconda3/etc/profile.d/conda.sh && conda activate testbe
             container_id: Container ID
             force: Force kill without graceful stop (use on timeout)
         """
-        # Release GPUs used by container before stop
-        gpu_ids = self._container_gpu_map.get(container_id)
-        if gpu_ids:
-            release_gpu(gpu_ids, self.logger)
-            del self._container_gpu_map[container_id]
-            self.logger.debug(f"Container {container_id[:12]} released GPUs {gpu_ids}")
-        
-        if force:
-            # Force kill container (no wait)
-            subprocess.run(['docker', 'kill', container_id], capture_output=True)
-        else:
-            # Graceful stop
-            subprocess.run(['docker', 'stop', container_id], capture_output=True)
-        subprocess.run(['docker', 'rm', container_id], capture_output=True)
+        try:
+            # Release GPUs used by container before stop
+            self._release_container_gpu(container_id)
+
+            if force:
+                # Force kill container (no wait)
+                subprocess.run(['docker', 'kill', container_id], capture_output=True)
+            else:
+                # Graceful stop
+                subprocess.run(['docker', 'stop', container_id], capture_output=True)
+            subprocess.run(['docker', 'rm', container_id], capture_output=True)
+        finally:
+            self._unregister_container_id(container_id)

{featurebench-0.2.0 → featurebench-0.2.2}/featurebench/harness/container.py

@@ -190,6 +190,7 @@ class EvalContainerManager:
         gpu_ids: str | None = None,
         proxy_port: int | None = None,
         docker_runtime_config: dict | None = None,
+        labels: dict[str, str] | None = None,
     ) -> Container:
         """
         Create and start a Docker container for evaluation.
@@ -201,6 +202,7 @@ class EvalContainerManager:
             gpu_ids: Comma-separated GPU IDs to use
             proxy_port: Proxy port for network access (uses bridge network with Docker host gateway)
             docker_runtime_config: Runtime config from repo_settings (need_gpu, shm_size, env_vars, env_exports, number_once)
+            labels: Docker labels to attach to the container
 
         Returns:
             Docker container object
@@ -297,6 +299,7 @@ class EvalContainerManager:
             environment=environment,
             network_mode=network_mode,
             shm_size=shm_size,
+            labels=labels,
         )
 
         # Check if GPU is available
@@ -365,5 +368,4 @@ class EvalContainerManager:
             container.remove()
             self.logger.info("Container removed successfully")
         except Exception as e:
-            self.logger.warning(f"Failed to cleanup container: {e}")
-
+            self.logger.warning(f"Failed to cleanup container: {e}")

{featurebench-0.2.0 → featurebench-0.2.2}/featurebench/harness/review_codes.py

@@ -135,6 +135,7 @@ def save_review_codes_level1(
     log_dir: Path,
     docker_image: str,
     logger: logging.Logger,
+    labels: dict[str, str] | None = None,
 ) -> None:
     """
     Save review codes for Level 1 instance.
@@ -176,6 +177,7 @@ def save_review_codes_level1(
             name=container_name,
             detach=True,
             remove=False,
+            labels=labels,
         )
 
         # Step 1: Restore project
@@ -290,6 +292,7 @@ def save_review_codes_level2(
     log_dir: Path,
     docker_image: str,
     logger: logging.Logger,
+    labels: dict[str, str] | None = None,
 ) -> None:
     """
     Save review codes for Level 2 instance.
@@ -332,6 +335,7 @@ def save_review_codes_level2(
             name=container_name,
             detach=True,
             remove=False,
+            labels=labels,
         )
 
         # Step 1: Clean /testbed/ and create README.md
@@ -402,4 +406,4 @@ def save_review_codes_level2(
                 container.stop(timeout=10)
                 container.remove()
             except Exception as e:
-                logger.warning(f"Failed to cleanup review container: {e}")
+                logger.warning(f"Failed to cleanup review container: {e}")