codeprobe 0.3.6__tar.gz → 0.3.8__tar.gz

{codeprobe-0.3.6 → codeprobe-0.3.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: codeprobe
-Version: 0.3.6
+Version: 0.3.8
 Summary: Benchmark AI coding agents against your own codebase. Mine real tasks from repo history, run agents, interpret results.
 Author: codeprobe contributors
 License-Expression: Apache-2.0

{codeprobe-0.3.6 → codeprobe-0.3.8}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "codeprobe"
-version = "0.3.6"
+version = "0.3.8"
 description = "Benchmark AI coding agents against your own codebase. Mine real tasks from repo history, run agents, interpret results."
 readme = "README.md"
 license = "Apache-2.0"

{codeprobe-0.3.6 → codeprobe-0.3.8}/src/codeprobe/__init__.py

@@ -1,3 +1,3 @@
 """codeprobe — Benchmark AI coding agents against your own codebase."""
 
-__version__ = "0.3.6"
+__version__ = "0.3.8"

{codeprobe-0.3.6 → codeprobe-0.3.8}/src/codeprobe/cli/init_cmd.py

@@ -153,17 +153,35 @@ def _detect_sourcegraph_in_mcp(
 
 
 def _prompt_sourcegraph_token() -> str:
-    """Prompt for Sourcegraph access token, checking env var first."""
-    import os
+    """Resolve Sourcegraph token: cached auth > env var > interactive prompt."""
+    from codeprobe.mining.sg_auth import AuthError, get_valid_token
+
+    # 1. Check cached auth and env var via the standard auth resolver
+    try:
+        cached = get_valid_token()
+        masked = cached.access_token[:4] + "..." + cached.access_token[-4:]
+        source = "cached" if not cached.refresh_token else "cached (refreshable)"
+        click.echo(f"  Found Sourcegraph credentials ({masked}, {source})")
+        if click.confirm("  Use these credentials?", default=True):
+            return cached.access_token
+    except AuthError:
+        pass
+
+    # 2. Offer OAuth-style flow via `codeprobe auth sourcegraph`
+    click.echo()
+    click.echo("  No cached Sourcegraph credentials found.")
+    click.echo("  Options:")
+    click.echo("    1. Paste a Personal Access Token now")
+    click.echo("    2. Run `codeprobe auth sourcegraph` first (recommended)")
+    choice = click.prompt("  Choose", type=click.IntRange(1, 2), default=1)
 
-    env_token = os.environ.get("SOURCEGRAPH_TOKEN", "")
-    if env_token:
-        masked = env_token[:4] + "..." + env_token[-4:] if len(env_token) > 8 else "***"
-        click.echo(f"  Found SOURCEGRAPH_TOKEN in environment ({masked})")
-        if click.confirm("  Use this token?", default=True):
-            return env_token
+    if choice == 2:
+        click.echo()
+        click.echo("  Run this command, then re-run `codeprobe init`:")
+        click.echo("    codeprobe auth sourcegraph")
+        raise SystemExit(0)
 
-    return click.prompt("Sourcegraph access token")
+    return click.prompt("  Sourcegraph access token", hide_input=True)
 
 
 def _prompt_sourcegraph_url() -> str | None:
@@ -176,6 +194,32 @@ def _prompt_sourcegraph_url() -> str | None:
     return url if url else None
 
 
+def _extract_sourcegraph_mcp(
+    discovered: list[tuple[Path, list[str]]],
+) -> dict | None:
+    """Load the Sourcegraph MCP config from a discovered config file.
+
+    Returns the full MCP config dict with only the Sourcegraph server,
+    or None if no Sourcegraph server is found.
+    """
+    import json
+
+    sg_names = {"sourcegraph", "sg", "sourcegraph-mcp"}
+    for path, server_names in discovered:
+        matching = [n for n in server_names if n.lower() in sg_names]
+        if not matching:
+            continue
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+        except (json.JSONDecodeError, OSError):
+            continue
+        servers = data.get("mcpServers", {})
+        for name in matching:
+            if name in servers:
+                return {"mcpServers": {name: servers[name]}}
+    return None
+
+
 def _goal_mcp(agents: list[str], name: str) -> _Result:
     """Goal 1: MCP comparison prompts."""
     agent = _prompt_agent(agents)
@@ -183,19 +227,27 @@ def _goal_mcp(agents: list[str], name: str) -> _Result:
 
     # Check if Sourcegraph is available in discovered MCP configs
     discovered = discover_mcp_configs()
-    use_sourcegraph = False
 
     if _detect_sourcegraph_in_mcp(discovered):
         click.echo()
         click.echo("Detected Sourcegraph MCP server in your configuration.")
-        click.echo("codeprobe can use the HTTP endpoint for better performance.")
-        use_sourcegraph = click.confirm("Use Sourcegraph HTTP MCP?", default=True)
-    else:
-        click.echo()
-        click.echo("Would you like to use Sourcegraph as the MCP server?")
-        use_sourcegraph = click.confirm("Use Sourcegraph?", default=False)
+        if click.confirm("Use this Sourcegraph config?", default=True):
+            sg_config = _extract_sourcegraph_mcp(discovered)
+            if sg_config:
+                return ask_mcp_comparison(
+                    experiment_name=name,
+                    agent=agent,
+                    model=model,
+                    mcp_config=sg_config,
+                )
+
+    click.echo()
+    click.echo("MCP server options:")
+    click.echo("  1. Use a Sourcegraph access token (PAT)")
+    click.echo("  2. Use an existing MCP config file")
+    choice = click.prompt("Choose", type=click.IntRange(1, 2), default=1)
 
-    if use_sourcegraph:
+    if choice == 1:
         token = _prompt_sourcegraph_token()
         sg_url = _prompt_sourcegraph_url()
         return ask_mcp_comparison(

{codeprobe-0.3.6 → codeprobe-0.3.8}/src/codeprobe/cli/mine_cmd.py

@@ -352,24 +352,31 @@ def _show_results_table(tasks: list["Task"]) -> None:
     click.echo()
 
 
-def _show_next_steps(repo_path: Path, min_files: int) -> None:
+def _show_next_steps(
+    repo_path: Path, min_files: int, *, llm_enriched: bool = False
+) -> None:
     """Phase 6: Show next steps."""
     click.echo("Next steps:")
     click.echo()
-    click.echo("  1. Review and enrich task instructions (recommended):")
-    click.echo("     codeprobe mine {path} --enrich".format(path=repo_path))
-    click.echo()
-    click.echo("  2. Run the eval:")
+    step = 1
+    if not llm_enriched:
+        click.echo(f"  {step}. Review and enrich task instructions (recommended):")
+        click.echo("     codeprobe mine {path} --enrich".format(path=repo_path))
+        click.echo()
+        step += 1
+    click.echo(f"  {step}. Run the eval:")
     click.echo("     codeprobe run {path} --agent claude".format(path=repo_path))
     click.echo()
-    click.echo("  3. Try a different model:")
+    step += 1
+    click.echo(f"  {step}. Try a different model:")
     click.echo(
         "     codeprobe run {path} --agent claude --model claude-sonnet-4-6".format(
             path=repo_path,
         )
     )
     click.echo()
-    click.echo("  4. Set a cost budget:")
+    step += 1
+    click.echo(f"  {step}. Set a cost budget:")
     click.echo(
         "     codeprobe run {path} --agent claude --max-cost-usd 5.00".format(
             path=repo_path,
@@ -558,6 +565,15 @@ def _interactive_config(
     )
 
 
+def _was_llm_used(no_llm: bool) -> bool:
+    """Check if LLM was available and used for instruction generation."""
+    if no_llm:
+        return False
+    from codeprobe.core.llm import llm_available
+
+    return llm_available()
+
+
 def _enrich_sdlc_tasks(
     tasks: list["Task"],
     mine_result: "MineResult",
@@ -1063,6 +1079,7 @@ def _dispatch_sdlc(
         return
 
     tasks = _enrich_sdlc_tasks(tasks, mine_result, no_llm, enrich)
+    llm_used = _was_llm_used(no_llm)
 
     tasks_dir = _clear_tasks_dir(repo_path)
     for task in tasks:
@@ -1078,6 +1095,7 @@ def _dispatch_sdlc(
         subsystems,
         repo_path,
         task_types=("sdlc_code_change",),
+        llm_enriched=llm_used,
     )
 
 
@@ -1255,7 +1273,7 @@ def _dispatch_mixed(
     if subsystems:
         click.echo(f"Subsystems: {', '.join(subsystems)}")
     click.echo()
-    _show_next_steps(repo_path, min_files)
+    _show_next_steps(repo_path, min_files, llm_enriched=_was_llm_used(no_llm))
 
 
 def _finish_mine_output(
@@ -1266,6 +1284,8 @@ def _finish_mine_output(
     subsystems: tuple[str, ...],
     repo_path: Path,
     task_types: tuple[str, ...] = (),
+    *,
+    llm_enriched: bool = False,
 ) -> None:
     """Shared output: quality warnings, path, subsystems, next steps."""
     from codeprobe.mining.writer import write_suite_manifest
@@ -1291,7 +1311,7 @@ def _finish_mine_output(
     if subsystems:
         click.echo(f"Subsystems: {', '.join(subsystems)}")
     click.echo()
-    _show_next_steps(repo_path, 0)
+    _show_next_steps(repo_path, 0, llm_enriched=llm_enriched)
 
 
 def run_mine(

{codeprobe-0.3.6 → codeprobe-0.3.8}/src/codeprobe/cli/wizard.py

@@ -45,26 +45,35 @@ def ask_mcp_comparison(
     agent: str,
     model: str | None,
     mcp_config_path: str | None = None,
+    mcp_config: dict | None = None,
     sourcegraph_token: str | None = None,
     sourcegraph_url: str | None = None,
 ) -> tuple[EvalrcConfig, list[ExperimentConfig]]:
     """Goal 1: Compare baseline agent vs MCP-augmented agent.
 
-    When *sourcegraph_token* is provided, generates an HTTP-based Sourcegraph
-    MCP config with an ``Authorization`` header and adds the ``sourcegraph``
-    preamble.  Otherwise falls back to loading the MCP config from
-    *mcp_config_path*.
+    Resolution order for MCP config:
+    1. *mcp_config* — pre-built dict (e.g. from discovered Claude Code config)
+    2. *sourcegraph_token* — build HTTP config with Authorization header
+    3. *mcp_config_path* — load from a JSON file on disk
     """
-    if sourcegraph_token is not None:
+    if mcp_config is not None:
+        mcp_data = mcp_config
+        # Detect if this is a Sourcegraph config for preamble
+        servers = mcp_data.get("mcpServers", {})
+        sg_names = {"sourcegraph", "sg", "sourcegraph-mcp"}
+        preambles: tuple[str, ...] = (
+            ("sourcegraph",) if any(k.lower() in sg_names for k in servers) else ()
+        )
+    elif sourcegraph_token is not None:
         mcp_data = build_sourcegraph_mcp_config(
             token=sourcegraph_token,
             url=sourcegraph_url or _DEFAULT_SOURCEGRAPH_URL,
         )
-        preambles: tuple[str, ...] = ("sourcegraph",)
+        preambles = ("sourcegraph",)
     else:
         if mcp_config_path is None:
             raise click.BadParameter(
-                "Either sourcegraph_token or mcp_config_path must be provided."
+                "Provide mcp_config, sourcegraph_token, or mcp_config_path."
             )
         mcp_data = _load_json(mcp_config_path)
         preambles = ()

{codeprobe-0.3.6 → codeprobe-0.3.8}/src/codeprobe/mining/extractor.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 import json as _json
 import logging
 import re
+import shlex
 import subprocess
 from dataclasses import dataclass, replace
 from pathlib import Path
@@ -104,6 +105,39 @@ def _get_changed_files(merge_sha: str, repo_path: Path) -> list[str]:
     return [f for f in result.stdout.strip().splitlines() if f.strip()]
 
 
+def _get_deleted_dirs(merge_sha: str, repo_path: Path) -> set[str]:
+    """Return directories that were entirely deleted in a merge commit.
+
+    Uses ``git diff --diff-filter=D --name-status`` to find deleted files,
+    then returns the set of parent directories where ALL files were deleted.
+    """
+    try:
+        result = subprocess.run(
+            [
+                "git",
+                "diff",
+                f"{merge_sha}^..{merge_sha}",
+                "--diff-filter=D",
+                "--name-only",
+            ],
+            cwd=str(repo_path),
+            capture_output=True,
+            text=True,
+            timeout=_DIFF_STAT_TIMEOUT,
+        )
+        if result.returncode != 0:
+            return set()
+    except (subprocess.TimeoutExpired, OSError):
+        return set()
+
+    deleted_files = [f for f in result.stdout.strip().splitlines() if f.strip()]
+    if not deleted_files:
+        return set()
+
+    # Collect parent directories of deleted files
+    return {str(Path(f).parent) for f in deleted_files}
+
+
 def extract_subsystems(
     prs: list[MergedPR],
     repo_path: Path,
@@ -156,26 +190,57 @@ _ISSUE_REF_PATTERN = re.compile(
 )
 
 
-def _build_test_command(language: str, test_files: list[str]) -> str:
+def _build_test_command(
+    language: str,
+    test_files: list[str],
+    repo_path: Path | None = None,
+    deleted_dirs: set[str] | None = None,
+) -> str:
     """Build a targeted test command from language and test file paths.
 
     Supports Python (pytest), Go (go test), and JS/TS (npm test with jest pattern).
     Falls back to the generic test.sh for unsupported languages or empty file lists.
+
+    When *repo_path* is provided and exists on disk, validates that referenced
+    paths exist in the target repo and drops any that don't.
+
+    When *deleted_dirs* is provided and ALL test packages fall within deleted
+    directories, generates a removal-verification command (checks dirs no longer
+    exist) instead of a test command.
     """
+    validate = repo_path is not None and repo_path.is_dir()
+
     if not test_files:
         return _DEFAULT_TEST_COMMAND
 
     if language == "python":
+        if validate:
+            test_files = [f for f in test_files if (repo_path / f).exists()]
+        if not test_files:
+            return _DEFAULT_TEST_COMMAND
         return f"pytest {' '.join(test_files)}"
 
     if language == "go":
-        # Extract unique package directories from test file paths
         packages = sorted({str(Path(f).parent) for f in test_files})
+
+        # Removal task: all test packages were deleted in this merge
+        if deleted_dirs and all(
+            any(pkg == d or pkg.startswith(d + "/") for d in deleted_dirs)
+            for pkg in packages
+        ):
+            checks = " && ".join(
+                f"test ! -d {shlex.quote('./' + pkg)}" for pkg in packages
+            )
+            return f"bash -c {shlex.quote(checks)}"
+
+        if validate:
+            packages = [p for p in packages if (repo_path / p).is_dir()]
+        if not packages:
+            return _DEFAULT_TEST_COMMAND
         go_paths = " ".join(f"./{pkg}/..." for pkg in packages)
         return f"go test {go_paths}"
 
     if language in ("javascript", "typescript"):
-        # Use the first test file's basename as the pattern
         pattern = Path(test_files[0]).name
         return f"npm test -- --testPathPattern={pattern}"
 
@@ -471,8 +536,13 @@ def extract_task_from_merge(
         )
         return None
 
-    # Build a verification command targeted to the detected test files and language
-    test_command = _build_test_command(language, test_files)
+    # Detect deleted directories for removal-task verification
+    deleted_dirs = _get_deleted_dirs(merge_sha, repo_path)
+
+    # Build a verification command targeted to the detected test files and language.
+    # Pass repo_path so missing packages (e.g. stripped vendor dirs) are filtered out.
+    # Pass deleted_dirs so removal tasks verify non-existence instead of compilation.
+    test_command = _build_test_command(language, test_files, repo_path, deleted_dirs)
 
     # Hard gate: stub test command means verification is meaningless — skip
     if test_command == _DEFAULT_TEST_COMMAND:

{codeprobe-0.3.6 → codeprobe-0.3.8}/src/codeprobe.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: codeprobe
-Version: 0.3.6
+Version: 0.3.8
 Summary: Benchmark AI coding agents against your own codebase. Mine real tasks from repo history, run agents, interpret results.
 Author: codeprobe contributors
 License-Expression: Apache-2.0

{codeprobe-0.3.6 → codeprobe-0.3.8}/tests/test_init_wizard.py

@@ -473,8 +473,8 @@ class TestInitCliIntegration:
 
         runner = CliRunner()
         # Inputs: goal=1, experiment name (enter=default), agent (enter=default),
-        # model (enter=skip), decline Sourcegraph, mcp config path
-        input_text = f"1\n\nclaude\n\nN\n{mcp_file}\n"
+        # model (enter=skip), choose=2 (MCP config file), mcp config path
+        input_text = f"1\n\nclaude\n\n2\n{mcp_file}\n"
         result = runner.invoke(main, ["init", str(tmp_path)], input=input_text)
         assert result.exit_code == 0, result.output
         assert not (tmp_path / ".evalrc.yaml").exists()
@@ -489,8 +489,8 @@ class TestInitCliIntegration:
 
         runner = CliRunner()
         # Inputs: goal=1, name=default, agent=claude, model=skip,
-        # use Sourcegraph=Y, token, url=default (enter)
-        input_text = "1\n\nclaude\n\nY\ntok_test123\n\n"
+        # choose=1 (PAT), choose=1 (paste now), token, url=default (enter)
+        input_text = "1\n\nclaude\n\n1\n1\ntok_test123\n\n"
         result = runner.invoke(main, ["init", str(tmp_path)], input=input_text)
         assert result.exit_code == 0, result.output
         assert not (tmp_path / ".evalrc.yaml").exists()

{codeprobe-0.3.6 → codeprobe-0.3.8}/tests/test_mining.py

@@ -617,6 +617,111 @@ class TestBuildTestCommand:
         result = _build_test_command("python", [])
         assert result == "bash tests/test.sh"
 
+    def test_go_filters_missing_packages(self, tmp_path: Path) -> None:
+        """Go packages that don't exist in repo_path are dropped."""
+        (tmp_path / "pkg" / "real").mkdir(parents=True)
+        result = _build_test_command(
+            "go",
+            ["pkg/real/foo_test.go", "pkg/missing/bar_test.go"],
+            repo_path=tmp_path,
+        )
+        assert "./pkg/real/..." in result
+        assert "missing" not in result
+
+    def test_go_all_missing_falls_back(self, tmp_path: Path) -> None:
+        """When all Go packages are missing, falls back to default."""
+        result = _build_test_command(
+            "go",
+            ["vendor/gone/x_test.go"],
+            repo_path=tmp_path,
+        )
+        assert result == "bash tests/test.sh"
+
+    def test_python_filters_missing_files(self, tmp_path: Path) -> None:
+        """Python test files that don't exist in repo_path are dropped."""
+        (tmp_path / "tests").mkdir()
+        (tmp_path / "tests" / "test_real.py").touch()
+        result = _build_test_command(
+            "python",
+            ["tests/test_real.py", "tests/test_gone.py"],
+            repo_path=tmp_path,
+        )
+        assert "test_real.py" in result
+        assert "test_gone.py" not in result
+
+    def test_no_repo_path_skips_validation(self) -> None:
+        """Without repo_path, all paths are kept (backward compat)."""
+        result = _build_test_command(
+            "go",
+            ["pkg/might/not/exist/x_test.go"],
+        )
+        assert "./pkg/might/not/exist/..." in result
+
+
+# ---------------------------------------------------------------------------
+# _build_test_command removal verification tests
+# ---------------------------------------------------------------------------
+
+
+class TestBuildTestCommandRemoval:
+    def test_go_removal_when_all_packages_deleted(self) -> None:
+        """When all test packages are in deleted dirs, generate removal check."""
+        result = _build_test_command(
+            "go",
+            ["cluster/images/etcd/migrate/migrate_test.go"],
+            deleted_dirs={"cluster/images/etcd/migrate"},
+        )
+        assert "test ! -d" in result
+        assert "go test" not in result
+        assert "cluster/images/etcd/migrate" in result
+
+    def test_go_removal_nested_deleted_dir(self) -> None:
+        """Packages under a deleted parent are detected as removal."""
+        result = _build_test_command(
+            "go",
+            [
+                "legacy/pkg/a/a_test.go",
+                "legacy/pkg/b/b_test.go",
+            ],
+            deleted_dirs={"legacy"},
+        )
+        assert "test ! -d" in result
+        assert "legacy/pkg/a" in result
+        assert "legacy/pkg/b" in result
+
+    def test_go_mixed_deleted_and_existing(self, tmp_path: Path) -> None:
+        """When some packages are deleted but others exist, use go test for survivors."""
+        (tmp_path / "pkg" / "alive").mkdir(parents=True)
+        result = _build_test_command(
+            "go",
+            ["pkg/alive/x_test.go", "pkg/dead/y_test.go"],
+            repo_path=tmp_path,
+            deleted_dirs={"pkg/dead"},
+        )
+        # Not all packages are deleted, so no removal check
+        assert "go test" in result
+        assert "pkg/alive" in result
+        # Dead package filtered by path validation
+        assert "pkg/dead" not in result
+
+    def test_go_no_deleted_dirs_normal_flow(self) -> None:
+        """Without deleted_dirs, normal go test command."""
+        result = _build_test_command(
+            "go",
+            ["pkg/auth/auth_test.go"],
+            deleted_dirs=None,
+        )
+        assert result == "go test ./pkg/auth/..."
+
+    def test_go_empty_deleted_dirs_normal_flow(self) -> None:
+        """Empty deleted_dirs set treated same as None."""
+        result = _build_test_command(
+            "go",
+            ["pkg/auth/auth_test.go"],
+            deleted_dirs=set(),
+        )
+        assert result == "go test ./pkg/auth/..."
+
 
 # ---------------------------------------------------------------------------
 # score_pr_quality tests
@@ -1113,6 +1218,10 @@ class TestRunMineClearsStale:
         stale_dir.mkdir(parents=True)
         (stale_dir / "instruction.md").write_text("stale")
 
+        # Create test file paths so path validation passes
+        (tmp_path / "tests").mkdir(exist_ok=True)
+        (tmp_path / "tests" / "test_auth.py").touch()
+
         merge_log = "aaaa1111bbbb2222 Merge pull request #1 from fix/auth\n"
         diff_files = "src/auth.py\ntests/test_auth.py\n"