codeprobe 0.2.2__tar.gz → 0.2.4__tar.gz

{codeprobe-0.2.2 → codeprobe-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: codeprobe
-Version: 0.2.2
+Version: 0.2.4
 Summary: Benchmark AI coding agents against your own codebase. Mine real tasks from repo history, run agents, interpret results.
 Author: codeprobe contributors
 License-Expression: Apache-2.0

{codeprobe-0.2.2 → codeprobe-0.2.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "codeprobe"
-version = "0.2.2"
+version = "0.2.4"
 description = "Benchmark AI coding agents against your own codebase. Mine real tasks from repo history, run agents, interpret results."
 readme = "README.md"
 license = "Apache-2.0"

{codeprobe-0.2.2 → codeprobe-0.2.4}/src/codeprobe/cli/run_cmd.py

@@ -183,10 +183,22 @@ def run_eval(
 
         click.echo(f"\nRunning config: {exp_config.label} ({len(task_dirs)} tasks)")
 
+        # Compute directories to exclude from git clean between sequential
+        # tasks so the experiment dir (untracked) isn't deleted.
+        _clean_excludes: tuple[str, ...] = ()
+        resolved_repo = Path(path).resolve()
+        try:
+            rel = exp_dir.resolve().relative_to(resolved_repo)
+            top_dir = str(rel).split("/")[0]
+            if top_dir and top_dir != ".":
+                _clean_excludes = (top_dir,)
+        except ValueError:
+            pass  # experiment dir is outside the repo
+
         results = execute_config(
             adapter=config_adapter,
             task_dirs=task_dirs,
-            repo_path=Path(path).resolve(),
+            repo_path=resolved_repo,
             experiment_config=exp_config,
             agent_config=agent_config,
             checkpoint_store=checkpoint_store,
@@ -195,6 +207,7 @@ def run_eval(
             max_cost_usd=max_cost_usd,
             parallel=parallel,
             repeats=repeats,
+            clean_excludes=_clean_excludes,
         )
 
         if owns_sandbox:

{codeprobe-0.2.2 → codeprobe-0.2.4}/src/codeprobe/core/executor.py

@@ -443,6 +443,7 @@ def execute_config(
     parallel: int = 1,
     isolation: IsolationStrategy | None = None,
     repeats: int = 1,
+    clean_excludes: tuple[str, ...] = (),
 ) -> list[CompletedTask]:
     """Execute all tasks for a single experiment configuration.
 
@@ -460,20 +461,6 @@ def execute_config(
     results are returned.  Tasks with ``unknown`` or ``subscription``
     cost models are skipped in accumulation.
     """
-    # Compute directories to exclude from git clean so that experiment
-    # artifacts (runs/, tasks/, experiment.json) survive between tasks.
-    # runs_dir is e.g. <repo>/mcp-comparison/runs/baseline — walk up to
-    # find the experiment root relative to repo_path.
-    _clean_excludes: tuple[str, ...] = ()
-    if runs_dir is not None:
-        try:
-            exp_root = runs_dir.resolve().parent.parent  # runs/<label> → exp dir
-            rel = exp_root.relative_to(repo_path.resolve())
-            # Exclude the top-level experiment directory name
-            _clean_excludes = (str(rel).split("/")[0],)
-        except ValueError:
-            pass  # experiment dir is outside the repo — nothing to exclude
-
     checkpointed_ids, results = _restore_checkpointed(checkpoint_store)
 
     # Filter checkpointed results to only include tasks in the current
@@ -578,7 +565,7 @@ def execute_config(
             # Reset working directory between tasks so leftovers from
             # task N don't corrupt task N+1's results.
             if idx > 0:
-                _git_reset_workdir(repo_path, extra_excludes=_clean_excludes)
+                _git_reset_workdir(repo_path, extra_excludes=clean_excludes)
             task_result = _run_one(task_dir, repeat_index=repeat_index)
             _handle_result(task_result)
     else:

{codeprobe-0.2.2 → codeprobe-0.2.4}/src/codeprobe/core/isolation.py

@@ -12,12 +12,28 @@ from typing import Protocol, runtime_checkable
 logger = logging.getLogger(__name__)
 
 
+def _discover_experiment_dirs(workdir: Path) -> list[str]:
+    """Find top-level directories that contain an experiment.json.
+
+    These are codeprobe experiment directories that must survive git clean.
+    """
+    excludes: list[str] = []
+    try:
+        for entry in workdir.iterdir():
+            if entry.is_dir() and (entry / "experiment.json").is_file():
+                excludes.append(entry.name)
+    except OSError:
+        pass
+    return excludes
+
+
 def git_restore_clean(workdir: Path, *, extra_excludes: tuple[str, ...] = ()) -> None:
     """Restore tracked files and remove untracked files in *workdir*.
 
     Uses ``git restore .`` (tolerant of empty diffs) followed by
-    ``git clean -fd``.  Always excludes ``.codeprobe`` and
-    ``.codeprobe-worktrees``; pass *extra_excludes* for more.
+    ``git clean -fd``.  Always excludes ``.codeprobe``,
+    ``.codeprobe-worktrees``, and any directories containing
+    ``experiment.json`` (codeprobe experiment dirs).
     """
     result = subprocess.run(
         ["git", "restore", "."],
@@ -39,6 +55,9 @@ def git_restore_clean(workdir: Path, *, extra_excludes: tuple[str, ...] = ()) ->
         "-e",
         ".codeprobe-worktrees",
     ]
+    # Auto-discover experiment directories inside the repo
+    for exp_dir in _discover_experiment_dirs(workdir):
+        clean_cmd += ["-e", exp_dir]
     for exc in extra_excludes:
         clean_cmd += ["-e", exc]
     subprocess.run(clean_cmd, cwd=workdir, check=True, capture_output=True)

{codeprobe-0.2.2 → codeprobe-0.2.4}/src/codeprobe.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: codeprobe
-Version: 0.2.2
+Version: 0.2.4
 Summary: Benchmark AI coding agents against your own codebase. Mine real tasks from repo history, run agents, interpret results.
 Author: codeprobe contributors
 License-Expression: Apache-2.0

{codeprobe-0.2.2 → codeprobe-0.2.4}/tests/test_executor.py

@@ -551,7 +551,8 @@ def test_execute_config_resets_workdir_between_sequential_tasks(tmp_path: Path):
         )
         # Reset should be called between tasks (not before first), so 2 times for 3 tasks
         assert mock_reset.call_count == 2
-        mock_reset.assert_any_call(Path("/repo"), extra_excludes=())
+        # First positional arg should be repo_path
+        assert mock_reset.call_args_list[0][0][0] == Path("/repo")
 
 
 def test_execute_config_no_reset_in_parallel_mode(tmp_path: Path):