agenttester 0.1.0__py3-none-any.whl

agenttester/__init__.py

@@ -0,0 +1,24 @@
+"""AgentTester: Multi-agent comparison tool.
+
+Usable as a CLI (``agenttester run …``), a Docker container, or a Python
+library::
+
+    from agenttester import Orchestrator, AgentConfig, load_config
+"""
+
+from .agent_runner import AgentResult, run_agent
+from .config import AgentConfig, load_config
+from .git_manager import DiffStats, GitManager
+from .orchestrator import Orchestrator
+
+__all__ = [
+    "AgentConfig",
+    "AgentResult",
+    "DiffStats",
+    "GitManager",
+    "Orchestrator",
+    "load_config",
+    "run_agent",
+]
+
+__version__ = "0.1.0"

agenttester/agent_runner.py

@@ -0,0 +1,341 @@
+"""Run a single agent process in a worktree."""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import shlex
+import signal
+import subprocess
+import tempfile
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+import rich.markup
+from rich.console import Console
+
+from .config import AgentConfig
+
+
+@dataclass
+class AgentResult:
+    """Result of a single agent run."""
+
+    agent_name: str
+    exit_code: int
+    duration: float
+    stdout: str
+    stderr: str
+    error: str | None = None
+
+
+# ── helpers for remote execution ──────────────────────────────────────
+
+
+def _rsync_to_remote(local: Path, host: str, remote_dir: str) -> None:
+    """Push a local directory to a remote host via rsync."""
+    subprocess.run(
+        [
+            "rsync",
+            "-az",
+            "--delete",
+            f"{local}/",
+            f"{host}:{remote_dir}/",
+        ],
+        check=True,
+        capture_output=True,
+    )
+
+
+def _rsync_from_remote(host: str, remote_dir: str, local: Path) -> None:
+    """Pull a remote directory back to local via rsync."""
+    subprocess.run(
+        [
+            "rsync",
+            "-az",
+            "--delete",
+            f"{host}:{remote_dir}/",
+            f"{local}/",
+        ],
+        check=True,
+        capture_output=True,
+    )
+
+
+def _build_ssh_command(
+    agent: AgentConfig,
+    remote_dir: str,
+    cmd: str,
+) -> str:
+    """Wrap *cmd* in an SSH invocation on the agent's host."""
+    env_exports = ""
+    if agent.env:
+        parts = " ".join(f"{k}={shlex.quote(v)}" for k, v in agent.env.items())
+        env_exports = f"export {parts} && "
+    inner = f"{env_exports}cd {shlex.quote(remote_dir)} && {cmd}"
+    return f"ssh {agent.host} {shlex.quote(inner)}"
+
+
+# ── core entry point ──────────────────────────────────────────────────
+
+
+def _prepare_command(agent: AgentConfig, prompt: str) -> tuple[str, Path | None, bool]:
+    """Substitute placeholders and decide stdin mode.
+
+    Returns (final_cmd, prompt_file_path | None, pipe_stdin).
+    """
+    cmd = agent.command
+    prompt_file_path: Path | None = None
+
+    if "{prompt_file}" in cmd:
+        fd, path_str = tempfile.mkstemp(suffix=".md", prefix="agenttester-prompt-")
+        prompt_file_path = Path(path_str)
+        with os.fdopen(fd, "w") as f:
+            f.write(prompt)
+        cmd = cmd.replace("{prompt_file}", path_str)
+
+    if "{prompt}" in cmd:
+        cmd = cmd.replace("{prompt}", shlex.quote(prompt))
+
+    has_placeholder = "{prompt}" in agent.command or "{prompt_file}" in agent.command
+    return cmd, prompt_file_path, not has_placeholder
+
+
+async def run_agent(
+    agent: AgentConfig,
+    worktree_path: Path,
+    prompt: str,
+    console: Console,
+    color: str,
+    output_lock: asyncio.Lock,
+) -> AgentResult:
+    """Run an agent locally or on a remote host."""
+    if agent.is_remote:
+        return await _run_remote(
+            agent, worktree_path, prompt, console, color, output_lock
+        )
+    return await _run_local(agent, worktree_path, prompt, console, color, output_lock)
+
+
+async def _run_local(
+    agent: AgentConfig,
+    worktree_path: Path,
+    prompt: str,
+    console: Console,
+    color: str,
+    output_lock: asyncio.Lock,
+) -> AgentResult:
+    """Run an agent as a local subprocess."""
+    start = time.monotonic()
+    cmd, prompt_file_path, pipe_stdin = _prepare_command(agent, prompt)
+
+    full_env = os.environ.copy()
+    full_env.update(agent.env)
+
+    prefix = f"[{color}]\\[{agent.name}][/{color}]"
+    stdout_lines: list[str] = []
+    stderr_lines: list[str] = []
+
+    proc: asyncio.subprocess.Process | None = None
+    try:
+        proc = await asyncio.create_subprocess_shell(
+            cmd,
+            stdin=asyncio.subprocess.PIPE if pipe_stdin else None,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+            cwd=worktree_path,
+            env=full_env,
+            start_new_session=True,
+        )
+
+        if pipe_stdin and proc.stdin:
+            proc.stdin.write(prompt.encode())
+            await proc.stdin.drain()
+            proc.stdin.close()
+
+        await _stream_and_wait(
+            proc,
+            agent.timeout,
+            stdout_lines,
+            stderr_lines,
+            console,
+            prefix,
+            output_lock,
+        )
+
+        return AgentResult(
+            agent_name=agent.name,
+            exit_code=proc.returncode or 0,
+            duration=time.monotonic() - start,
+            stdout="\n".join(stdout_lines),
+            stderr="\n".join(stderr_lines),
+        )
+
+    except TimeoutError:
+        _kill_proc_tree(proc)
+        return AgentResult(
+            agent_name=agent.name,
+            exit_code=-1,
+            duration=time.monotonic() - start,
+            stdout="\n".join(stdout_lines),
+            stderr="\n".join(stderr_lines),
+            error=f"Timed out after {agent.timeout}s",
+        )
+    except Exception as e:
+        _kill_proc_tree(proc)
+        return AgentResult(
+            agent_name=agent.name,
+            exit_code=-1,
+            duration=time.monotonic() - start,
+            stdout="\n".join(stdout_lines),
+            stderr="\n".join(stderr_lines),
+            error=str(e),
+        )
+    finally:
+        if prompt_file_path:
+            prompt_file_path.unlink(missing_ok=True)
+
+
+async def _run_remote(
+    agent: AgentConfig,
+    worktree_path: Path,
+    prompt: str,
+    console: Console,
+    color: str,
+    output_lock: asyncio.Lock,
+) -> AgentResult:
+    """Rsync to remote, run agent via SSH, rsync results back."""
+    start = time.monotonic()
+    remote_dir = f"{agent.remote_workdir}/{agent.name}"
+    prefix = f"[{color}]\\[{agent.name}][/{color}]"
+    stdout_lines: list[str] = []
+    stderr_lines: list[str] = []
+
+    try:
+        # 1. Push worktree to remote
+        async with output_lock:
+            console.print(f"  {prefix} [dim]syncing to {agent.host}:{remote_dir}[/dim]")
+        await asyncio.to_thread(_rsync_to_remote, worktree_path, agent.host, remote_dir)
+
+        # 2. Build and run command over SSH
+        cmd, prompt_file_path, _pipe_stdin = _prepare_command(agent, prompt)
+        ssh_cmd = _build_ssh_command(agent, remote_dir, cmd)
+
+        proc = await asyncio.create_subprocess_shell(
+            ssh_cmd,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+            start_new_session=True,
+        )
+
+        await _stream_and_wait(
+            proc,
+            agent.timeout,
+            stdout_lines,
+            stderr_lines,
+            console,
+            prefix,
+            output_lock,
+        )
+        exit_code = proc.returncode or 0
+
+        # 3. Pull results back
+        async with output_lock:
+            console.print(
+                f"  {prefix} [dim]syncing from {agent.host}:{remote_dir}[/dim]"
+            )
+        await asyncio.to_thread(
+            _rsync_from_remote, agent.host, remote_dir, worktree_path
+        )
+
+        if prompt_file_path:
+            prompt_file_path.unlink(missing_ok=True)
+
+        return AgentResult(
+            agent_name=agent.name,
+            exit_code=exit_code,
+            duration=time.monotonic() - start,
+            stdout="\n".join(stdout_lines),
+            stderr="\n".join(stderr_lines),
+        )
+
+    except TimeoutError:
+        return AgentResult(
+            agent_name=agent.name,
+            exit_code=-1,
+            duration=time.monotonic() - start,
+            stdout="\n".join(stdout_lines),
+            stderr="\n".join(stderr_lines),
+            error=f"Timed out after {agent.timeout}s",
+        )
+    except Exception as e:
+        return AgentResult(
+            agent_name=agent.name,
+            exit_code=-1,
+            duration=time.monotonic() - start,
+            stdout="\n".join(stdout_lines),
+            stderr="\n".join(stderr_lines),
+            error=str(e),
+        )
+
+
+# ── shared streaming helper ───────────────────────────────────────────
+
+
+async def _stream_and_wait(
+    proc: asyncio.subprocess.Process,
+    timeout: int,
+    stdout_lines: list[str],
+    stderr_lines: list[str],
+    console: Console,
+    prefix: str,
+    output_lock: asyncio.Lock,
+) -> None:
+    """Stream stdout/stderr and wait, raising TimeoutError on expiry."""
+
+    async def _read(
+        stream: asyncio.StreamReader | None,
+        lines: list[str],
+        is_err: bool,
+    ) -> None:
+        if stream is None:
+            return
+        while True:
+            raw = await stream.readline()
+            if not raw:
+                break
+            line = raw.decode("utf-8", errors="replace").rstrip()
+            lines.append(line)
+            # Escape Rich markup in agent output to avoid parse errors
+            safe = rich.markup.escape(line)
+            async with output_lock:
+                if is_err:
+                    console.print(f"  {prefix} [dim]{safe}[/dim]")
+                else:
+                    console.print(f"  {prefix} {safe}")
+
+    try:
+        await asyncio.wait_for(
+            asyncio.gather(
+                _read(proc.stdout, stdout_lines, False),
+                _read(proc.stderr, stderr_lines, True),
+            ),
+            timeout=timeout,
+        )
+        await proc.wait()
+    except asyncio.TimeoutError as exc:
+        _kill_proc_tree(proc)
+        raise TimeoutError from exc
+
+
+def _kill_proc_tree(
+    proc: asyncio.subprocess.Process | None,
+) -> None:
+    """Kill a process and its entire process group."""
+    if proc is None or proc.returncode is not None:
+        return
+    try:
+        os.killpg(proc.pid, signal.SIGKILL)
+    except (ProcessLookupError, PermissionError):
+        proc.kill()

agenttester/cli.py

@@ -0,0 +1,177 @@
+"""CLI entry point."""
+
+from __future__ import annotations
+
+import asyncio
+import urllib.error
+from pathlib import Path
+from typing import Annotated
+
+import typer
+from rich.console import Console
+
+from .config import load_config
+from .orchestrator import Orchestrator
+from .repl import run_repl
+from .vllm import query as _vllm_query
+
+app = typer.Typer(
+    name="agenttester",
+    help="Send a prompt to multiple coding agents in parallel and compare results.",
+    no_args_is_help=True,
+)
+console = Console()
+
+
+def _parse_agent_names(raw: list[str]) -> list[str]:
+    """Flatten comma-separated and repeated --agents values."""
+    names: list[str] = []
+    for entry in raw:
+        names.extend(n.strip() for n in entry.split(",") if n.strip())
+    return names
+
+
+@app.command()
+def run(
+    prompt: Annotated[
+        str | None, typer.Argument(help="Prompt to send to each agent")
+    ] = None,
+    agents: Annotated[
+        list[str] | None,
+        typer.Option(
+            "--agents",
+            "-a",
+            help="Agent names (comma-separated or repeated)",
+        ),
+    ] = None,
+    prompt_file: Annotated[
+        Path | None,
+        typer.Option("--prompt-file", "-f", help="Read prompt from a file"),
+    ] = None,
+    keep_worktrees: Annotated[
+        bool,
+        typer.Option("--keep-worktrees", help="Keep worktrees after the run"),
+    ] = False,
+    config: Annotated[
+        Path | None,
+        typer.Option("--config", "-c", help="Path to config YAML file"),
+    ] = None,
+    timeout: Annotated[
+        int | None,
+        typer.Option(
+            "--timeout",
+            "-t",
+            help="Override timeout for all agents (seconds)",
+        ),
+    ] = None,
+    repo: Annotated[
+        Path | None,
+        typer.Option("--repo", "-r", help="Path to target git repo (default: cwd)"),
+    ] = None,
+) -> None:
+    """Run agents in parallel on a prompt and compare results."""
+    # Resolve prompt
+    if prompt_file:
+        if not prompt_file.exists():
+            console.print(f"[red]Prompt file not found: {prompt_file}[/red]")
+            raise typer.Exit(1)
+        prompt_text = prompt_file.read_text().strip()
+    elif prompt:
+        prompt_text = prompt
+    else:
+        console.print("[red]Provide a prompt or --prompt-file[/red]")
+        raise typer.Exit(1)
+
+    # Resolve agents
+    if not agents:
+        console.print("[red]Specify at least one agent with --agents[/red]")
+        raise typer.Exit(1)
+
+    agent_names = _parse_agent_names(agents)
+    if len(agent_names) > 5:
+        console.print("[red]Maximum 5 agents allowed[/red]")
+        raise typer.Exit(1)
+
+    # Load config and resolve agent objects
+    all_agents = load_config(config)
+    selected = []
+    for name in agent_names:
+        if name not in all_agents:
+            console.print(
+                f"[red]Unknown agent: {name}[/red]\n"
+                f"Available: {', '.join(sorted(all_agents))}"
+            )
+            raise typer.Exit(1)
+        agent_cfg = all_agents[name]
+        if timeout is not None:
+            agent_cfg.timeout = timeout
+        selected.append(agent_cfg)
+
+    # Run
+    repo_path = (repo or Path.cwd()).resolve()
+    orchestrator = Orchestrator(repo_path, console)
+
+    try:
+        asyncio.run(
+            orchestrator.run(prompt_text, selected, keep_worktrees=keep_worktrees)
+        )
+    except RuntimeError as e:
+        console.print(f"[red]{e}[/red]")
+        raise typer.Exit(1) from e
+
+
+@app.command()
+def query(
+    endpoint: Annotated[
+        str, typer.Argument(help="vLLM server endpoint (http://HOST:PORT)")
+    ],
+    model_id: Annotated[str, typer.Argument(help="Model ID served by the endpoint")],
+    prompt: Annotated[str, typer.Argument(help="Prompt to send")],
+    max_tokens: Annotated[
+        int, typer.Option("--max-tokens", help="Maximum tokens to generate")
+    ] = 2048,
+) -> None:
+    """Query a vLLM model server and print the response."""
+    try:
+        result = _vllm_query(
+            endpoint, model_id, [{"role": "user", "content": prompt}], max_tokens
+        )
+        console.print(result)
+    except urllib.error.HTTPError as e:
+        body = e.read().decode(errors="replace")
+        console.print(f"[red]HTTP {e.code}: {body}[/red]")
+        raise typer.Exit(1) from e
+    except OSError as e:
+        console.print(f"[red]{e}[/red]")
+        raise typer.Exit(1) from e
+
+
+@app.command()
+def repl(
+    config: Annotated[
+        Path | None,
+        typer.Option("--config", "-c", help="Path to config YAML file"),
+    ] = None,
+) -> None:
+    """Start an interactive REPL across all vLLM model agents."""
+    asyncio.run(run_repl(config))
+
+
+@app.command("agents")
+def list_agents(
+    config: Annotated[
+        Path | None,
+        typer.Option("--config", "-c", help="Path to config YAML file"),
+    ] = None,
+) -> None:
+    """List available agents."""
+    all_agents = load_config(config)
+    console.print("[bold]Available agents:[/bold]\n")
+    for name, agent in sorted(all_agents.items()):
+        preset_names = ("claude", "aider", "codex")
+        tag = "[dim](preset)[/dim]" if name in preset_names else ""
+        console.print(f"  [bold]{name}[/bold] {tag}")
+        console.print(f"    command: [dim]{agent.command}[/dim]")
+        console.print(f"    host:    {agent.host}")
+        console.print(f"    commit:  {agent.commit_style}  timeout: {agent.timeout}s")
+        console.print()

agenttester/config.py

@@ -0,0 +1,65 @@
+"""Agent configuration and loading."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import yaml
+
+from .presets import PRESETS
+
+CONFIG_CANDIDATES = [
+    "agenttester.yaml",
+    "agenttester.yml",
+    ".agenttester.yaml",
+    ".agenttester.yml",
+]
+
+
+@dataclass
+class AgentConfig:
+    """Configuration for a single coding agent."""
+
+    name: str
+    command: str
+    host: str = "localhost"
+    remote_workdir: str = "/tmp/agenttester"
+    commit_style: str = "auto"  # "auto" (agent commits) or "manual" (we commit)
+    env: dict[str, str] = field(default_factory=dict)
+    timeout: int = 600  # seconds
+
+    @property
+    def is_remote(self) -> bool:
+        """True when the agent runs on a non-local host."""
+        return self.host != "localhost"
+
+
+def load_config(config_path: Path | None = None) -> dict[str, AgentConfig]:
+    """Load agent configs from YAML, merged with built-in presets."""
+    agents: dict[str, AgentConfig] = {}
+    for name, preset in PRESETS.items():
+        agents[name] = AgentConfig(name=name, **preset)
+
+    if config_path is None:
+        for candidate in CONFIG_CANDIDATES:
+            p = Path(candidate)
+            if p.exists():
+                config_path = p
+                break
+
+    if config_path and config_path.exists():
+        with open(config_path) as f:
+            data = yaml.safe_load(f) or {}
+        for name, agent_data in (data.get("agents") or {}).items():
+            agents[name] = AgentConfig(
+                name=name,
+                command=agent_data["command"],
+                host=agent_data.get("host", "localhost"),
+                remote_workdir=agent_data.get("remote_workdir", "/tmp/agenttester"),
+                commit_style=agent_data.get("commit_style", "auto"),
+                env=agent_data.get("env", {}),
+                timeout=agent_data.get("timeout", 600),
+            )
+
+    return agents

agenttester/git_manager.py

@@ -0,0 +1,123 @@
+"""Git worktree and branch management."""
+
+from __future__ import annotations
+
+import contextlib
+import os
+import re
+import shlex
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import git
+from git.exc import GitCommandError
+
+
+@dataclass
+class DiffStats:
+    """Diff statistics between base ref and an agent's branch."""
+
+    files_changed: int = 0
+    insertions: int = 0
+    deletions: int = 0
+    changed_files: list[str] = field(default_factory=list)
+
+
+class GitManager:
+    """Manages git worktrees and branches for parallel agent runs."""
+
+    def __init__(self, repo_path: Path) -> None:
+        self.repo_path = repo_path
+        self.worktree_base = repo_path / ".agenttester" / "worktrees"
+        self.repo = git.Repo(repo_path)
+        self._apply_env(self.repo)
+
+    @staticmethod
+    def _apply_env(repo: git.Repo) -> None:
+        """Inherit the full shell environment so SSH keys and config are available."""
+        env = {k: v for k, v in os.environ.items() if k.isidentifier()}
+        if "GIT_SSH_COMMAND" not in env:
+            ssh_config = Path.home() / ".ssh" / "config"
+            if ssh_config.exists():
+                env["GIT_SSH_COMMAND"] = f"ssh -F {shlex.quote(str(ssh_config))}"
+
+        repo.git.update_environment(**env)
+
+    def has_commits(self) -> bool:
+        """Check if the repo has at least one commit."""
+        try:
+            self.repo.git.rev_parse("HEAD")
+        except GitCommandError:
+            return False
+        return True
+
+    def get_head_ref(self) -> str:
+        """Return the current HEAD commit SHA."""
+        return self.repo.head.commit.hexsha
+
+    def create_worktree(self, agent_name: str, run_id: str) -> Path:
+        """Create a worktree with a new branch for an agent run."""
+        branch = f"agenttester/{run_id}/{agent_name}"
+        worktree_path = self.worktree_base / run_id / agent_name
+        worktree_path.parent.mkdir(parents=True, exist_ok=True)
+        self.repo.git.worktree("add", "-b", branch, str(worktree_path))
+        return worktree_path
+
+    def commit_all(self, worktree_path: Path, agent_name: str) -> bool:
+        """Stage and commit all changes in a worktree.
+
+        Returns True if a commit was created.
+        """
+        wt_repo = git.Repo(worktree_path)
+        self._apply_env(wt_repo)
+        wt_repo.git.add("-A")
+        if not wt_repo.index.diff("HEAD"):
+            return False
+        wt_repo.index.commit(f"agenttester: {agent_name} changes")
+        return True
+
+    def get_diff_stats(self, run_id: str, agent_name: str, base_ref: str) -> DiffStats:
+        """Get diff statistics between the base ref and an agent's branch."""
+        branch = f"agenttester/{run_id}/{agent_name}"
+        try:
+            stat_line = self.repo.git.diff("--shortstat", base_ref, branch)
+
+            files_changed = insertions = deletions = 0
+            if stat_line:
+                if m := re.search(r"(\d+) file", stat_line):
+                    files_changed = int(m.group(1))
+                if m := re.search(r"(\d+) insertion", stat_line):
+                    insertions = int(m.group(1))
+                if m := re.search(r"(\d+) deletion", stat_line):
+                    deletions = int(m.group(1))
+
+            name_only = self.repo.git.diff("--name-only", base_ref, branch)
+            changed_files = [f for f in name_only.strip().split("\n") if f]
+
+            return DiffStats(
+                files_changed=files_changed,
+                insertions=insertions,
+                deletions=deletions,
+                changed_files=changed_files,
+            )
+        except GitCommandError:
+            return DiffStats()
+
+    def cleanup_worktree(self, run_id: str, agent_name: str) -> None:
+        """Remove a single worktree."""
+        worktree_path = self.worktree_base / run_id / agent_name
+        if worktree_path.exists():
+            self.repo.git.worktree("remove", str(worktree_path), "--force")
+
+    def cleanup_run(self, run_id: str) -> None:
+        """Remove all worktrees for a run. Branches are preserved."""
+        run_dir = self.worktree_base / run_id
+        if not run_dir.exists():
+            return
+        for agent_dir in sorted(run_dir.iterdir()):
+            if agent_dir.is_dir():
+                with contextlib.suppress(GitCommandError):
+                    self.repo.git.worktree("remove", str(agent_dir), "--force")
+        for d in (run_dir, self.worktree_base):
+            with contextlib.suppress(OSError):
+                d.rmdir()

agenttester/orchestrator.py

@@ -0,0 +1,135 @@
+"""Orchestrate parallel agent runs."""
+
+from __future__ import annotations
+
+import asyncio
+import uuid
+from pathlib import Path
+
+from rich.console import Console
+
+from .agent_runner import AgentResult, run_agent
+from .config import AgentConfig
+from .git_manager import GitManager
+from .report import generate_report
+
+AGENT_COLORS = ["cyan", "green", "yellow", "magenta", "blue"]
+MAX_CONCURRENT = 5
+
+
+class Orchestrator:
+    """Run multiple agents in parallel, each in its own worktree."""
+
+    def __init__(self, repo_path: Path, console: Console) -> None:
+        self.repo_path = repo_path
+        self.git = GitManager(repo_path)
+        self.console = console
+        self.semaphore = asyncio.Semaphore(MAX_CONCURRENT)
+
+    async def run(
+        self,
+        prompt: str,
+        agents: list[AgentConfig],
+        *,
+        keep_worktrees: bool = False,
+    ) -> list[AgentResult]:
+        """Execute a prompt across all agents and produce a comparison report."""
+        if not self.git.has_commits():
+            msg = (
+                "Repository has no commits. "
+                "Create an initial commit before running agents."
+            )
+            raise RuntimeError(msg)
+
+        if len(agents) > MAX_CONCURRENT:
+            msg = f"Maximum {MAX_CONCURRENT} agents allowed, got {len(agents)}"
+            raise RuntimeError(msg)
+
+        run_id = uuid.uuid4().hex[:8]
+        base_ref = self.git.get_head_ref()
+
+        self.console.print(
+            f"[bold]Starting run [cyan]{run_id}[/cyan] "
+            f"with {len(agents)} agent(s) from [dim]{base_ref[:12]}[/dim][/bold]\n"
+        )
+
+        # Create worktrees
+        worktrees: dict[str, Path] = {}
+        for agent in agents:
+            try:
+                wt = self.git.create_worktree(agent.name, run_id)
+                worktrees[agent.name] = wt
+                self.console.print(f"  [dim]worktree ready:[/dim] {agent.name} → {wt}")
+            except Exception as e:
+                self.console.print(
+                    f"  [red]Failed to create worktree for {agent.name}: {e}[/red]"
+                )
+
+        self.console.print()
+
+        # Run agents concurrently
+        output_lock = asyncio.Lock()
+
+        async def _run_one(agent: AgentConfig, color: str) -> AgentResult:
+            wt = worktrees.get(agent.name)
+            if not wt:
+                return AgentResult(
+                    agent.name, -1, 0.0, "", "", "Worktree creation failed"
+                )
+            async with self.semaphore:
+                result = await run_agent(
+                    agent, wt, prompt, self.console, color, output_lock
+                )
+            # Auto-commit for agents that don't commit themselves
+            if agent.commit_style == "manual" and result.exit_code == 0:
+                try:
+                    committed = self.git.commit_all(wt, agent.name)
+                    if committed:
+                        self.console.print(
+                            f"  [dim]Auto-committed changes for {agent.name}[/dim]"
+                        )
+                except Exception as e:
+                    self.console.print(
+                        f"  [yellow]Warning: auto-commit failed for "
+                        f"{agent.name}: {e}[/yellow]"
+                    )
+            return result
+
+        tasks = [
+            _run_one(agent, AGENT_COLORS[i % len(AGENT_COLORS)])
+            for i, agent in enumerate(agents)
+        ]
+        raw_results = await asyncio.gather(*tasks, return_exceptions=True)
+
+        # Normalize results
+        results: list[AgentResult] = []
+        for i, r in enumerate(raw_results):
+            if isinstance(r, BaseException):
+                results.append(AgentResult(agents[i].name, -1, 0.0, "", "", str(r)))
+            else:
+                results.append(r)
+
+        # Print summary
+        self.console.print("\n[bold]Results:[/bold]")
+        for r in results:
+            icon = "✅" if r.exit_code == 0 else "❌"
+            self.console.print(
+                f"  {icon} [bold]{r.agent_name}[/bold] "
+                f"— {r.duration:.1f}s, exit {r.exit_code}"
+                + (f" ({r.error})" if r.error else "")
+            )
+
+        # Generate report
+        report = generate_report(run_id, base_ref, prompt, results, self.git)
+        report_path = self.repo_path / f"agenttester-report-{run_id}.md"
+        report_path.write_text(report)
+        self.console.print(f"\n[bold]Report:[/bold] {report_path}")
+
+        # Cleanup
+        if keep_worktrees:
+            self.console.print("[dim]Worktrees kept for inspection.[/dim]")
+        else:
+            self.git.cleanup_run(run_id)
+            self.console.print("[dim]Worktrees removed. Branches preserved.[/dim]")
+
+        return results

agenttester/presets.py

@@ -0,0 +1,25 @@
+"""Built-in agent command presets."""
+
+from __future__ import annotations
+
+PRESETS: dict[str, dict] = {
+    "claude": {
+        "command": (
+            "claude -p {prompt}"
+            ' --allowedTools "Bash,Read,Edit"'
+            " --permission-mode acceptEdits"
+        ),
+        "commit_style": "auto",
+        "timeout": 600,
+    },
+    "aider": {
+        "command": "aider --yes-always --no-auto-commits --message {prompt}",
+        "commit_style": "manual",
+        "timeout": 600,
+    },
+    "codex": {
+        "command": "codex exec --sandbox danger-full-access {prompt}",
+        "commit_style": "auto",
+        "timeout": 600,
+    },
+}

agenttester/repl.py

@@ -0,0 +1,121 @@
+"""Interactive multi-model REPL with persistent conversation history."""
+
+from __future__ import annotations
+
+import asyncio
+import re
+import urllib.error
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import yaml
+from rich.console import Console
+from rich.panel import Panel
+from rich.prompt import Prompt
+
+from .config import CONFIG_CANDIDATES
+from .vllm import query as _vllm_query
+
+_COMMAND_PATTERN = re.compile(
+    r"agenttester\s+query\s+(https?://\S+)\s+(\S+)\s+\{prompt\}"
+)
+
+
+@dataclass
+class Model:
+    name: str
+    endpoint: str
+    model_id: str
+    messages: list[dict] = field(default_factory=list)
+
+
+def load_models(config_path: Path | None = None) -> dict[str, Model]:
+    if config_path is None:
+        for candidate in CONFIG_CANDIDATES:
+            p = Path(candidate)
+            if p.exists():
+                config_path = p
+                break
+
+    if not config_path or not config_path.exists():
+        return {}
+
+    with open(config_path) as f:
+        data = yaml.safe_load(f) or {}
+
+    models: dict[str, Model] = {}
+    for name, agent_data in (data.get("agents") or {}).items():
+        m = _COMMAND_PATTERN.search(agent_data.get("command", ""))
+        if m:
+            models[name] = Model(name=name, endpoint=m.group(1), model_id=m.group(2))
+
+    return models
+
+
+def _query_sync(model: Model, prompt: str, max_tokens: int = 2048) -> str:
+    model.messages.append({"role": "user", "content": prompt})
+    try:
+        reply = _vllm_query(model.endpoint, model.model_id, model.messages, max_tokens)
+    except urllib.error.HTTPError as e:
+        body = e.read().decode(errors="replace")
+        model.messages.pop()
+        return f"[error] HTTP {e.code}: {body}"
+    except OSError as e:
+        model.messages.pop()
+        return f"[error] {e}"
+    model.messages.append({"role": "assistant", "content": reply})
+    return reply
+
+
+async def _query_all(models: dict[str, Model], prompt: str) -> dict[str, str]:
+    tasks = {
+        name: asyncio.to_thread(_query_sync, model, prompt)
+        for name, model in models.items()
+    }
+    results = await asyncio.gather(*tasks.values(), return_exceptions=True)
+    return {
+        name: str(r) if isinstance(r, Exception) else r
+        for name, r in zip(tasks.keys(), results, strict=True)
+    }
+
+
+async def run_repl(config_path: Path | None = None) -> None:
+    console = Console()
+    models = load_models(config_path)
+    if not models:
+        console.print("[red]No vLLM model agents found in config.[/red]")
+        console.print(
+            "Add agents using 'agenttester query' commands to your agenttester.yaml."
+        )
+        return
+
+    console.print(f"[bold]Models:[/bold] {', '.join(models)}")
+    console.print(
+        "[dim]Commands: /reset (clear history), exit or Ctrl-C to quit[/dim]\n"
+    )
+
+    while True:
+        try:
+            prompt = Prompt.ask("[bold cyan]>[/bold cyan]")
+        except (EOFError, KeyboardInterrupt):
+            console.print("\n[dim]bye[/dim]")
+            break
+
+        prompt = prompt.strip()
+        if not prompt:
+            continue
+        if prompt == "exit":
+            break
+        if prompt == "/reset":
+            for model in models.values():
+                model.messages.clear()
+            console.print("[dim]Context cleared.[/dim]\n")
+            continue
+
+        console.print()
+        responses = await _query_all(models, prompt)
+        for name, reply in responses.items():
+            console.print(
+                Panel(reply, title=f"[bold]{name}[/bold]", border_style="blue")
+            )
+        console.print()

agenttester/report.py

@@ -0,0 +1,75 @@
+"""Generate markdown comparison reports."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from .agent_runner import AgentResult
+from .git_manager import GitManager
+
+
+def generate_report(
+    run_id: str,
+    base_ref: str,
+    prompt: str,
+    results: list[AgentResult],
+    git: GitManager,
+) -> str:
+    """Build a markdown report comparing agent results."""
+    lines: list[str] = [
+        f"# AgentTester Report: {run_id}",
+        "",
+        f"**Base ref**: `{base_ref[:12]}`",
+        f"**Date**: {datetime.now(tz=timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')}",
+        f"**Agents**: {', '.join(r.agent_name for r in results)}",
+        "",
+        "## Prompt",
+        "",
+        "```",
+        prompt,
+        "```",
+        "",
+        "## Summary",
+        "",
+        "| Agent | Status | Duration | Files | Insertions | Deletions |",
+        "|-------|--------|----------|-------|------------|-----------|",
+    ]
+
+    for r in results:
+        stats = git.get_diff_stats(run_id, r.agent_name, base_ref)
+        status = "✅" if r.exit_code == 0 else "❌"
+        if r.error:
+            status += f" {r.error}"
+        lines.append(
+            f"| {r.agent_name} | {status} | {r.duration:.1f}s "
+            f"| {stats.files_changed} | +{stats.insertions} "
+            f"| -{stats.deletions} |"
+        )
+
+    lines.append("")
+
+    # Per-agent details
+    for r in results:
+        stats = git.get_diff_stats(run_id, r.agent_name, base_ref)
+
+        lines.extend(
+            [
+                f"## {r.agent_name}",
+                "",
+                f"**Branch**: `agenttester/{run_id}/{r.agent_name}`",
+                f"**Duration**: {r.duration:.1f}s",
+                f"**Exit code**: {r.exit_code}",
+            ]
+        )
+
+        if r.error:
+            lines.append(f"**Error**: {r.error}")
+
+        if stats.changed_files:
+            lines.extend(["", "### Files Changed", ""])
+            for f in stats.changed_files:
+                lines.append(f"- `{f}`")
+
+        lines.append("")
+
+    return "\n".join(lines)

agenttester/vllm.py

@@ -0,0 +1,35 @@
+"""HTTP client for vLLM OpenAI-compatible inference servers."""
+
+from __future__ import annotations
+
+import json
+import urllib.error
+import urllib.request
+
+
+def query(
+    endpoint: str,
+    model_id: str,
+    messages: list[dict],
+    max_tokens: int = 2048,
+    timeout: int = 120,
+) -> str:
+    """Send a chat completion request and return the response text.
+
+    Raises urllib.error.HTTPError or OSError on failure.
+    """
+    payload = json.dumps(
+        {
+            "model": model_id,
+            "messages": messages,
+            "max_tokens": max_tokens,
+        }
+    ).encode()
+    req = urllib.request.Request(
+        f"{endpoint.rstrip('/')}/v1/chat/completions",
+        data=payload,
+        headers={"Content-Type": "application/json"},
+    )
+    with urllib.request.urlopen(req, timeout=timeout) as resp:
+        data = json.loads(resp.read())
+    return data["choices"][0]["message"]["content"]

agenttester-0.1.0.dist-info/METADATA

@@ -0,0 +1,15 @@
+Metadata-Version: 2.4
+Name: agenttester
+Version: 0.1.0
+Summary: Send prompts to multiple coding agents in parallel and compare results
+License-Expression: MIT
+License-File: LICENSE
+Requires-Python: >=3.10
+Requires-Dist: gitpython>=3.1
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: rich>=13.0
+Requires-Dist: typer>=0.9
+Provides-Extra: dev
+Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.4; extra == 'dev'

agenttester-0.1.0.dist-info/RECORD

@@ -0,0 +1,15 @@
+agenttester/__init__.py,sha256=auNtvZAfdLaH5OG43PwjKq-TuUFxAUj_EB7h51pVNGM,551
+agenttester/agent_runner.py,sha256=h6etvHJ2lFHA5i5wQyFUpC2Pdlo3zYxcqODfkXMRiqI,9939
+agenttester/cli.py,sha256=YoX-GvqXokk9Q0dXOO7S5-02ZZAVloAaL5BgnR8h6yA,5422
+agenttester/config.py,sha256=1TvBiY6YqkKNPpZUOAfCWuvqemEJ7KiD4rWUHU9dRVU,1938
+agenttester/git_manager.py,sha256=0brFkO0kaRtyL0FL-9vcOwoeas8sb5mZ1oGgIvvp_vg,4513
+agenttester/orchestrator.py,sha256=1TyvjvY8FbeGJylbUSPRP_NCkaQGY1dqsjboiJLNYiE,4771
+agenttester/presets.py,sha256=WAYXJDJD6iKY4ga8rCfeZ23MbB2UY20beQJ4NR5fie0,649
+agenttester/repl.py,sha256=Lg3De4IwWVs6eJETQIGSArhOyBf_vmagrb6c7JWnrf4,3615
+agenttester/report.py,sha256=C1CtErdsQYUh7c6Xxfg_FW9ZVEyTnbZ0i8aeiK-vXIg,2084
+agenttester/vllm.py,sha256=h3abkbHWxqCW0evNOzG68PhTxzz4ore-rlGKAX8ug64,917
+agenttester-0.1.0.dist-info/METADATA,sha256=rUBqZbXVWTjD_U_P_C9ocTac1htkNvuzgmDg4zQAxvY,470
+agenttester-0.1.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+agenttester-0.1.0.dist-info/entry_points.txt,sha256=KUIZqIbLGFCwo_dG9bfLGTO52aFGeoIcAMmapqozvMM,52
+agenttester-0.1.0.dist-info/licenses/LICENSE,sha256=t1iGtmPs5Z0ZhwTcbgnw4nUIDhoDPWlOLwSIw5GLJKo,1066
+agenttester-0.1.0.dist-info/RECORD,,

agenttester-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.29.0
+Root-Is-Purelib: true
+Tag: py3-none-any

agenttester-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+agenttester = agenttester.cli:app

agenttester-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 sroomberg
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.