codeprobe 0.1.7__tar.gz → 0.2.1__tar.gz

codeprobe-0.2.1/PKG-INFO

@@ -0,0 +1,224 @@
+Metadata-Version: 2.4
+Name: codeprobe
+Version: 0.2.1
+Summary: Benchmark AI coding agents against your own codebase. Mine real tasks from repo history, run agents, interpret results.
+Author: codeprobe contributors
+License-Expression: Apache-2.0
+Project-URL: Homepage, https://github.com/sjarmak/codeprobe
+Project-URL: Repository, https://github.com/sjarmak/codeprobe
+Project-URL: Issues, https://github.com/sjarmak/codeprobe/issues
+Keywords: ai,benchmark,eval,coding-agent,mcp
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: click<9,>=8.0
+Requires-Dist: pyyaml<7,>=6.0
+Requires-Dist: anthropic>=0.39
+Requires-Dist: openai>=1.66
+Requires-Dist: tiktoken<1,>=0.7
+Requires-Dist: scipy<2,>=1.11
+Provides-Extra: dev
+Requires-Dist: pytest<9,>=8.0; extra == "dev"
+Requires-Dist: pytest-cov<6,>=5.0; extra == "dev"
+Requires-Dist: ruff<1,>=0.4; extra == "dev"
+Requires-Dist: mypy<2,>=1.10; extra == "dev"
+Requires-Dist: types-PyYAML<7,>=6.0; extra == "dev"
+Requires-Dist: scipy<2,>=1.11; extra == "dev"
+Dynamic: license-file
+
+# codeprobe
+
+Benchmark AI coding agents against **your own codebase**.
+
+Mine real tasks from your repo history, run agents against them, and find out which setup actually works best for YOUR code — not someone else's benchmark suite.
+
+## Why codeprobe?
+
+Existing benchmarks (SWE-bench, HumanEval) use fixed task sets that AI models may have memorized from training data. codeprobe mines tasks from **your private repo history**, producing benchmarks that are impossible to contaminate.
+
+## Prerequisites
+
+codeprobe orchestrates external AI coding agents — you need at least one installed:
+
+| Agent              | Install                                          | Required env var                |
+| ------------------ | ------------------------------------------------ | ------------------------------- |
+| **Claude Code**    | [claude.ai/download](https://claude.ai/download) | `ANTHROPIC_API_KEY`             |
+| **GitHub Copilot** | `npm install -g @github/copilot-cli` (>= 1.0.4)  | GitHub auth via `gh auth login` |
+| **Codex**          | Included via `pip install codeprobe[codex]`      | `OPENAI_API_KEY`                |
+
+You also need:
+
+- **Python 3.11+**
+- **Git** (for task mining and worktree isolation)
+- **GitHub CLI** (`gh`) — optional, for mining tasks from GitHub PRs with linked issues
+
+The `assess` and `mine --enrich` commands need an LLM for scoring/enrichment. codeprobe auto-detects the best available backend:
+
+| Priority | Backend       | Install                                          | Env var             |
+| -------- | ------------- | ------------------------------------------------ | ------------------- |
+| 1        | Anthropic SDK | `pip install codeprobe[anthropic]`               | `ANTHROPIC_API_KEY` |
+| 2        | OpenAI SDK    | `pip install codeprobe[codex]`                   | `OPENAI_API_KEY`    |
+| 3        | Claude CLI    | [claude.ai/download](https://claude.ai/download) | `ANTHROPIC_API_KEY` |
+
+Override with `CODEPROBE_LLM_BACKEND=anthropic|openai|claude-cli`. Without any backend, `assess` falls back to heuristic scoring.
+
+## Quick Start
+
+```bash
+pip install codeprobe
+
+cd /path/to/your/repo
+
+codeprobe assess .      # Score benchmarking potential (optional)
+codeprobe mine .        # Extract tasks from repo history
+codeprobe run .         # Run agents against tasks
+codeprobe interpret .   # Get recommendations
+```
+
+## Commands
+
+| Command                  | Purpose                                          |
+| ------------------------ | ------------------------------------------------ |
+| `codeprobe assess`       | Score a codebase's benchmarking potential        |
+| `codeprobe init`         | Interactive wizard — choose what to compare      |
+| `codeprobe mine`         | Mine eval tasks from merged PRs/MRs              |
+| `codeprobe probe`        | Generate fast micro-benchmark probes (30s each)  |
+| `codeprobe experiment`   | Manage comparison experiments (init, add-config) |
+| `codeprobe run`          | Execute tasks against AI agents                  |
+| `codeprobe interpret`    | Analyze results, rank configurations             |
+| `codeprobe oracle-check` | Compare agent answer against oracle ground truth |
+| `codeprobe scaffold`     | Create/validate eval task directories            |
+| `codeprobe ratings`      | Record and analyze agent session quality ratings |
+
+## Two Ways to Generate Tasks
+
+### 1. SDLC Tasks (from merged PRs)
+
+Mine real code-change tasks from your git history. Agents must reproduce known fixes and features.
+
+```bash
+codeprobe mine . --count 10 --source github
+codeprobe mine . --count 5 --min-files 4    # Harder tasks (more files changed)
+codeprobe mine . --enrich                    # LLM-enriched instructions
+```
+
+### 2. Micro-Benchmark Probes
+
+Fast exact-match tasks (30s each) that test code navigation and comprehension — no agent sandbox needed.
+
+```bash
+codeprobe probe . -n 10 -l python -s 42 -o ./probes
+```
+
+Generates four probe types: find-function, count-callers, return-type, module-dependency.
+
+## MCP Comparison Experiments
+
+Compare agent performance with and without MCP tools (Sourcegraph, GitHub, etc.).
+
+### Mine org-scale comprehension tasks
+
+```bash
+# Set up Sourcegraph credentials
+export SOURCEGRAPH_TOKEN="your-token"
+
+# Mine MCP-optimized tasks with Sourcegraph ground truth enrichment
+codeprobe mine /path/to/repo \
+  --org-scale --mcp-families --count 5 \
+  --no-interactive --no-llm \
+  --sg-repo github.com/sg-evals/your-repo
+```
+
+MCP task families: `symbol-reference-trace`, `type-hierarchy-consumers`, `change-scope-audit`.
+
+### Set up the experiment
+
+```bash
+# Create experiment
+codeprobe experiment init /path/to/repo --name mcp-comparison
+
+# Copy mined tasks into the experiment
+cp -r /path/to/repo/.codeprobe/tasks/* /path/to/repo/mcp-comparison/tasks/
+
+# Baseline config (no MCP, no preamble)
+codeprobe experiment add-config /path/to/repo/mcp-comparison \
+  --label baseline --agent claude --model claude-haiku-4-5-20251001
+
+# Sourcegraph MCP config (preamble + MCP server)
+codeprobe experiment add-config /path/to/repo/mcp-comparison \
+  --label with-sourcegraph --agent claude --model claude-haiku-4-5-20251001 \
+  --preamble sourcegraph \
+  --mcp-config '{"mcpServers":{"sourcegraph":{"type":"http","url":"https://sourcegraph.com/.api/mcp/v1","headers":{"Authorization":"token $SOURCEGRAPH_TOKEN"}}}}'
+
+# Run and interpret
+codeprobe run /path/to/repo/mcp-comparison --agent claude --max-cost-usd 5.00
+codeprobe interpret /path/to/repo/mcp-comparison
+```
+
+### Preambles
+
+Preambles are composable instruction templates prepended to the agent's prompt for MCP-enabled configs. Built-in preambles: `sourcegraph`, `github`.
+
+Override built-ins by placing a `.md` file in:
+
+- `<task_dir>/preambles/` (per-task)
+- `.codeprobe/preambles/` (project-level)
+- `~/.codeprobe/preambles/` (user-level)
+
+Template variables: `{{sg_repo}}`, `{{repo_name}}`, `{{repo_path}}`, `{{task_id}}`
+
+## Key Flags
+
+```bash
+# Running
+codeprobe run . --parallel 5          # Run 5 tasks concurrently (worktree-isolated)
+codeprobe run . --max-cost-usd 2.00   # Stop when cost budget is reached
+codeprobe run . --dry-run             # Estimate resource usage without running
+
+# Mining
+codeprobe mine . --enrich             # Use LLM to improve weak task instructions
+codeprobe mine . --org-scale          # Mine comprehension tasks (not SDLC)
+codeprobe mine . --mcp-families       # Include MCP-optimized task families
+codeprobe mine . --sg-repo REPO       # Sourcegraph repo for ground truth enrichment
+
+# Experiment configs
+codeprobe experiment add-config . --preamble sourcegraph  # Attach MCP preamble
+codeprobe experiment add-config . --mcp-config config.json  # Attach MCP server
+
+# Output
+codeprobe interpret . --format csv    # Export for pivot tables
+codeprobe interpret . --format html   # Self-contained HTML report
+```
+
+## Supported Agents
+
+- **Claude Code** (`--agent claude`) — headless via `claude -p`
+- **GitHub Copilot** (`--agent copilot`) — via Copilot CLI
+- **Codex** (`--agent codex`) — via OpenAI API
+- Custom agents via the `AgentAdapter` protocol
+
+## Supported Git Hosts
+
+GitHub, GitLab, Bitbucket, Azure DevOps, Gitea/Forgejo, and local repos.
+
+## Configuration
+
+Create a `.evalrc.yaml` in your repo root:
+
+```yaml
+name: my-experiment
+agents: [claude, copilot]
+models: [claude-sonnet-4-6, claude-opus-4-6]
+tasks_dir: .codeprobe/tasks
+```
+
+## License
+
+Apache-2.0

codeprobe-0.2.1/README.md

@@ -0,0 +1,189 @@
+# codeprobe
+
+Benchmark AI coding agents against **your own codebase**.
+
+Mine real tasks from your repo history, run agents against them, and find out which setup actually works best for YOUR code — not someone else's benchmark suite.
+
+## Why codeprobe?
+
+Existing benchmarks (SWE-bench, HumanEval) use fixed task sets that AI models may have memorized from training data. codeprobe mines tasks from **your private repo history**, producing benchmarks that are impossible to contaminate.
+
+## Prerequisites
+
+codeprobe orchestrates external AI coding agents — you need at least one installed:
+
+| Agent              | Install                                          | Required env var                |
+| ------------------ | ------------------------------------------------ | ------------------------------- |
+| **Claude Code**    | [claude.ai/download](https://claude.ai/download) | `ANTHROPIC_API_KEY`             |
+| **GitHub Copilot** | `npm install -g @github/copilot-cli` (>= 1.0.4)  | GitHub auth via `gh auth login` |
+| **Codex**          | Included via `pip install codeprobe[codex]`      | `OPENAI_API_KEY`                |
+
+You also need:
+
+- **Python 3.11+**
+- **Git** (for task mining and worktree isolation)
+- **GitHub CLI** (`gh`) — optional, for mining tasks from GitHub PRs with linked issues
+
+The `assess` and `mine --enrich` commands need an LLM for scoring/enrichment. codeprobe auto-detects the best available backend:
+
+| Priority | Backend       | Install                                          | Env var             |
+| -------- | ------------- | ------------------------------------------------ | ------------------- |
+| 1        | Anthropic SDK | `pip install codeprobe[anthropic]`               | `ANTHROPIC_API_KEY` |
+| 2        | OpenAI SDK    | `pip install codeprobe[codex]`                   | `OPENAI_API_KEY`    |
+| 3        | Claude CLI    | [claude.ai/download](https://claude.ai/download) | `ANTHROPIC_API_KEY` |
+
+Override with `CODEPROBE_LLM_BACKEND=anthropic|openai|claude-cli`. Without any backend, `assess` falls back to heuristic scoring.
+
+## Quick Start
+
+```bash
+pip install codeprobe
+
+cd /path/to/your/repo
+
+codeprobe assess .      # Score benchmarking potential (optional)
+codeprobe mine .        # Extract tasks from repo history
+codeprobe run .         # Run agents against tasks
+codeprobe interpret .   # Get recommendations
+```
+
+## Commands
+
+| Command                  | Purpose                                          |
+| ------------------------ | ------------------------------------------------ |
+| `codeprobe assess`       | Score a codebase's benchmarking potential        |
+| `codeprobe init`         | Interactive wizard — choose what to compare      |
+| `codeprobe mine`         | Mine eval tasks from merged PRs/MRs              |
+| `codeprobe probe`        | Generate fast micro-benchmark probes (30s each)  |
+| `codeprobe experiment`   | Manage comparison experiments (init, add-config) |
+| `codeprobe run`          | Execute tasks against AI agents                  |
+| `codeprobe interpret`    | Analyze results, rank configurations             |
+| `codeprobe oracle-check` | Compare agent answer against oracle ground truth |
+| `codeprobe scaffold`     | Create/validate eval task directories            |
+| `codeprobe ratings`      | Record and analyze agent session quality ratings |
+
+## Two Ways to Generate Tasks
+
+### 1. SDLC Tasks (from merged PRs)
+
+Mine real code-change tasks from your git history. Agents must reproduce known fixes and features.
+
+```bash
+codeprobe mine . --count 10 --source github
+codeprobe mine . --count 5 --min-files 4    # Harder tasks (more files changed)
+codeprobe mine . --enrich                    # LLM-enriched instructions
+```
+
+### 2. Micro-Benchmark Probes
+
+Fast exact-match tasks (30s each) that test code navigation and comprehension — no agent sandbox needed.
+
+```bash
+codeprobe probe . -n 10 -l python -s 42 -o ./probes
+```
+
+Generates four probe types: find-function, count-callers, return-type, module-dependency.
+
+## MCP Comparison Experiments
+
+Compare agent performance with and without MCP tools (Sourcegraph, GitHub, etc.).
+
+### Mine org-scale comprehension tasks
+
+```bash
+# Set up Sourcegraph credentials
+export SOURCEGRAPH_TOKEN="your-token"
+
+# Mine MCP-optimized tasks with Sourcegraph ground truth enrichment
+codeprobe mine /path/to/repo \
+  --org-scale --mcp-families --count 5 \
+  --no-interactive --no-llm \
+  --sg-repo github.com/sg-evals/your-repo
+```
+
+MCP task families: `symbol-reference-trace`, `type-hierarchy-consumers`, `change-scope-audit`.
+
+### Set up the experiment
+
+```bash
+# Create experiment
+codeprobe experiment init /path/to/repo --name mcp-comparison
+
+# Copy mined tasks into the experiment
+cp -r /path/to/repo/.codeprobe/tasks/* /path/to/repo/mcp-comparison/tasks/
+
+# Baseline config (no MCP, no preamble)
+codeprobe experiment add-config /path/to/repo/mcp-comparison \
+  --label baseline --agent claude --model claude-haiku-4-5-20251001
+
+# Sourcegraph MCP config (preamble + MCP server)
+codeprobe experiment add-config /path/to/repo/mcp-comparison \
+  --label with-sourcegraph --agent claude --model claude-haiku-4-5-20251001 \
+  --preamble sourcegraph \
+  --mcp-config '{"mcpServers":{"sourcegraph":{"type":"http","url":"https://sourcegraph.com/.api/mcp/v1","headers":{"Authorization":"token $SOURCEGRAPH_TOKEN"}}}}'
+
+# Run and interpret
+codeprobe run /path/to/repo/mcp-comparison --agent claude --max-cost-usd 5.00
+codeprobe interpret /path/to/repo/mcp-comparison
+```
+
+### Preambles
+
+Preambles are composable instruction templates prepended to the agent's prompt for MCP-enabled configs. Built-in preambles: `sourcegraph`, `github`.
+
+Override built-ins by placing a `.md` file in:
+
+- `<task_dir>/preambles/` (per-task)
+- `.codeprobe/preambles/` (project-level)
+- `~/.codeprobe/preambles/` (user-level)
+
+Template variables: `{{sg_repo}}`, `{{repo_name}}`, `{{repo_path}}`, `{{task_id}}`
+
+## Key Flags
+
+```bash
+# Running
+codeprobe run . --parallel 5          # Run 5 tasks concurrently (worktree-isolated)
+codeprobe run . --max-cost-usd 2.00   # Stop when cost budget is reached
+codeprobe run . --dry-run             # Estimate resource usage without running
+
+# Mining
+codeprobe mine . --enrich             # Use LLM to improve weak task instructions
+codeprobe mine . --org-scale          # Mine comprehension tasks (not SDLC)
+codeprobe mine . --mcp-families       # Include MCP-optimized task families
+codeprobe mine . --sg-repo REPO       # Sourcegraph repo for ground truth enrichment
+
+# Experiment configs
+codeprobe experiment add-config . --preamble sourcegraph  # Attach MCP preamble
+codeprobe experiment add-config . --mcp-config config.json  # Attach MCP server
+
+# Output
+codeprobe interpret . --format csv    # Export for pivot tables
+codeprobe interpret . --format html   # Self-contained HTML report
+```
+
+## Supported Agents
+
+- **Claude Code** (`--agent claude`) — headless via `claude -p`
+- **GitHub Copilot** (`--agent copilot`) — via Copilot CLI
+- **Codex** (`--agent codex`) — via OpenAI API
+- Custom agents via the `AgentAdapter` protocol
+
+## Supported Git Hosts
+
+GitHub, GitLab, Bitbucket, Azure DevOps, Gitea/Forgejo, and local repos.
+
+## Configuration
+
+Create a `.evalrc.yaml` in your repo root:
+
+```yaml
+name: my-experiment
+agents: [claude, copilot]
+models: [claude-sonnet-4-6, claude-opus-4-6]
+tasks_dir: .codeprobe/tasks
+```
+
+## License
+
+Apache-2.0

{codeprobe-0.1.7 → codeprobe-0.2.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "codeprobe"
-version = "0.1.7"
+version = "0.2.1"
 description = "Benchmark AI coding agents against your own codebase. Mine real tasks from repo history, run agents, interpret results."
 readme = "README.md"
 license = "Apache-2.0"

{codeprobe-0.1.7 → codeprobe-0.2.1}/src/codeprobe/__init__.py

@@ -1,3 +1,3 @@
 """codeprobe — Benchmark AI coding agents against your own codebase."""
 
-__version__ = "0.1.7"
+__version__ = "0.2.0"

{codeprobe-0.1.7 → codeprobe-0.2.1}/src/codeprobe/adapters/claude.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import json
 import re
+import shutil
 import subprocess
 import tempfile
 from pathlib import Path
@@ -69,16 +70,31 @@ class ClaudeAdapter(BaseAdapter):
 
         mcp_path = self._write_mcp_config(config)
         if mcp_path:
-            cmd.extend(["--mcp-config", mcp_path])
+            cmd.extend(["--mcp-config", mcp_path, "--strict-mcp-config"])
 
         return cmd
 
     def isolate_session(self, slot_id: int) -> dict[str, str]:
-        """Return a per-slot CLAUDE_CONFIG_DIR for session isolation."""
+        """Return a per-slot CLAUDE_CONFIG_DIR for session isolation.
+
+        Copies authentication credentials from the real ``~/.claude/``
+        directory so the agent subprocess can authenticate.
+        """
         config_dir = (
             Path(tempfile.gettempdir()) / "codeprobe-claude" / f"slot-{slot_id}"
         )
         config_dir.mkdir(parents=True, exist_ok=True)
+
+        # Copy auth credentials from the user's real config dir.
+        # Without these the subprocess gets "Not logged in".
+        real_config = Path.home() / ".claude"
+        if real_config.is_dir():
+            for name in ("credentials.json", ".credentials.json"):
+                src = real_config / name
+                dst = config_dir / name
+                if src.is_file():
+                    shutil.copy2(src, dst)
+
         return {"CLAUDE_CONFIG_DIR": str(config_dir)}
 
     def parse_output(

{codeprobe-0.1.7 → codeprobe-0.2.1}/src/codeprobe/cli/__init__.py

@@ -1,18 +1,89 @@
 """CLI entry point for codeprobe."""
 
+import json as _json
+import logging
+import sys
+
 import click
 
 from codeprobe import __version__
 
 
+class _JsonFormatter(logging.Formatter):
+    """Emit one JSON object per log line."""
+
+    def format(self, record: logging.LogRecord) -> str:
+        payload = {
+            "level": record.levelname,
+            "logger": record.name,
+            "message": record.getMessage(),
+            "timestamp": self.formatTime(record, "%Y-%m-%dT%H:%M:%S%z"),
+        }
+        return _json.dumps(payload)
+
+
+def _configure_logging(verbose: int, quiet: bool, log_format: str = "text") -> None:
+    """Configure namespace-scoped logging for codeprobe.* modules.
+
+    Attaches a StreamHandler to `logging.getLogger("codeprobe")` so that
+    all 26+ codeprobe.* modules emit through hierarchy without touching
+    third-party loggers (httpx, urllib3, etc.).
+    """
+    if quiet:
+        level = logging.WARNING
+    elif verbose >= 1:
+        level = logging.DEBUG
+    else:
+        level = logging.INFO
+
+    logger = logging.getLogger("codeprobe")
+    logger.setLevel(level)
+    logger.propagate = False  # don't bubble to root
+
+    # Idempotent: tests / repeat invocations must not duplicate handlers.
+    for h in list(logger.handlers):
+        logger.removeHandler(h)
+
+    handler = logging.StreamHandler(sys.stderr)
+    if log_format == "json":
+        handler.setFormatter(_JsonFormatter())
+    elif verbose >= 1:
+        fmt = "%(levelname)s %(name)s: %(message)s"
+        handler.setFormatter(logging.Formatter(fmt))
+    else:
+        fmt = "%(levelname)s: %(message)s"
+        handler.setFormatter(logging.Formatter(fmt))
+    logger.addHandler(handler)
+
+
 @click.group()
+@click.option(
+    "-v",
+    "--verbose",
+    count=True,
+    help="Increase log verbosity (-v sets DEBUG).",
+)
+@click.option(
+    "-q",
+    "--quiet",
+    is_flag=True,
+    default=False,
+    help="Suppress INFO logs (WARNING and above only).",
+)
+@click.option(
+    "--log-format",
+    type=click.Choice(["text", "json"]),
+    default="text",
+    help="Log output format (default: text). 'json' emits one JSON object per line.",
+)
 @click.version_option(version=__version__, prog_name="codeprobe")
-def main() -> None:
+def main(verbose: int, quiet: bool, log_format: str) -> None:
     """Benchmark AI coding agents against your own codebase.
 
     Mine real tasks from your repo history, run agents against them,
     and interpret the results to find which setup works best for YOUR code.
     """
+    _configure_logging(verbose=verbose, quiet=quiet, log_format=log_format)
 
 
 @main.command()
@@ -121,6 +192,20 @@ def init(path: str) -> None:
     default=False,
     help="Run LLM verification on curated ground truth.",
 )
+@click.option(
+    "--mcp-families",
+    is_flag=True,
+    default=False,
+    help="Include MCP-advantaged task families (symbol-reference-trace, "
+    "type-hierarchy-consumers, change-scope-audit). Only with --org-scale.",
+)
+@click.option(
+    "--sg-repo",
+    default="",
+    help="Sourcegraph repo identifier for ground truth enrichment "
+    "(e.g. github.com/sg-evals/numpy). Defaults to github.com/sg-evals/{repo_name} "
+    "when --mcp-families is used. Requires SOURCEGRAPH_TOKEN env var.",
+)
 def mine(
     path: str,
     count: int,
@@ -139,6 +224,8 @@ def mine(
     curate: bool,
     backends: tuple[str, ...],
     verify_curation_flag: bool,
+    mcp_families: bool,
+    sg_repo: str,
 ) -> None:
     """Mine eval tasks from a repository's history.
 
@@ -175,6 +262,8 @@ def mine(
         curate=curate,
         backends=backends,
         verify_curation_flag=verify_curation_flag,
+        mcp_families=mcp_families,
+        sg_repo=sg_repo,
     )
 
 
@@ -272,6 +361,20 @@ def init_experiment(path: str, name: str, description: str) -> None:
 @click.option(
     "--mcp-config", default=None, help="MCP config as JSON string or file path."
 )
+@click.option(
+    "--instruction-variant",
+    default=None,
+    help="Instruction file variant (e.g., instruction_mcp.md). Default: instruction.md.",
+)
+@click.option(
+    "--preamble",
+    "preambles",
+    multiple=True,
+    help=(
+        "Preamble to prepend to the instruction. Repeatable. "
+        "Built-ins: sourcegraph, github. Or path to a custom .md file."
+    ),
+)
 def add_config(
     path: str,
     label: str,
@@ -279,6 +382,8 @@ def add_config(
     model: str | None,
     permission_mode: str,
     mcp_config: str | None,
+    instruction_variant: str | None,
+    preambles: tuple[str, ...],
 ) -> None:
     """Add a configuration to an existing experiment."""
     from codeprobe.cli.experiment_cmd import experiment_add_config
@@ -290,6 +395,8 @@ def add_config(
         model=model,
         permission_mode=permission_mode,
         mcp_config_str=mcp_config,
+        instruction_variant=instruction_variant,
+        preambles=preambles,
     )
 
 

{codeprobe-0.1.7 → codeprobe-0.2.1}/src/codeprobe/cli/experiment_cmd.py

@@ -63,6 +63,8 @@ def experiment_add_config(
     model: str | None,
     permission_mode: str,
     mcp_config_str: str | None,
+    instruction_variant: str | None = None,
+    preambles: tuple[str, ...] = (),
 ) -> None:
     """Add a configuration to an existing experiment."""
     exp_dir = Path(path)
@@ -104,6 +106,8 @@ def experiment_add_config(
         model=model,
         permission_mode=permission_mode,
         mcp_config=mcp_config,
+        instruction_variant=instruction_variant,
+        preambles=preambles,
     )
 
     # Validate the label is a safe path component