agenttester 0.1.0__tar.gz

agenttester-0.1.0/.dockerignore

@@ -0,0 +1,9 @@
+.git
+.agenttester
+_plans
+__pycache__
+*.egg-info
+dist
+.venv
+*.pyc
+agenttester-report-*.md

agenttester-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,19 @@
+name: CI
+
+on:
+  push:
+    branches: ["**"]
+  pull_request:
+  workflow_call:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v4
+      - run: uv venv
+      - run: uv pip install -e ".[dev]"
+      - run: uv run ruff check src/ tests/
+      - run: uv run ruff format --check src/ tests/
+      - run: uv run pytest

agenttester-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,27 @@
+name: Publish to PyPI
+
+on:
+  push:
+    tags:
+      - "v*"
+
+permissions:
+  id-token: write  # trusted publishing (no API token needed)
+
+jobs:
+  test:
+    uses: ./.github/workflows/ci.yml
+
+  publish:
+    needs: test
+    runs-on: ubuntu-latest
+    environment: pypi
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v4
+
+      - name: Build package
+        run: uv build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

agenttester-0.1.0/.gitignore

@@ -0,0 +1,8 @@
+_plans/
+.agenttester/
+agenttester-report-*.md
+__pycache__/
+*.egg-info/
+dist/
+*.pyc
+.venv/

agenttester-0.1.0/Dockerfile

@@ -0,0 +1,22 @@
+FROM python:3.12-slim
+
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        git \
+        openssh-client \
+        rsync \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN git config --global user.email "agenttester@container" \
+    && git config --global user.name "agenttester"
+
+WORKDIR /app
+COPY . /app
+RUN uv pip install --system --no-cache .
+
+# Mount the target repo at /repo
+VOLUME ["/repo"]
+WORKDIR /repo
+
+ENTRYPOINT ["agenttester"]

agenttester-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 sroomberg
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

agenttester-0.1.0/PKG-INFO

@@ -0,0 +1,15 @@
+Metadata-Version: 2.4
+Name: agenttester
+Version: 0.1.0
+Summary: Send prompts to multiple coding agents in parallel and compare results
+License-Expression: MIT
+License-File: LICENSE
+Requires-Python: >=3.10
+Requires-Dist: gitpython>=3.1
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: rich>=13.0
+Requires-Dist: typer>=0.9
+Provides-Extra: dev
+Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.4; extra == 'dev'

agenttester-0.1.0/README.md

@@ -0,0 +1,115 @@
+# AgentTester
+
+> **⚠️ Experimental** — This project is under active development. APIs, config format, and CLI flags may change without notice.
+
+Send a single prompt to multiple coding agents running in parallel and compare the results. Each agent works in its own [git worktree](https://git-scm.com/docs/git-worktree) on a separate branch so they never interfere with each other.
+
+## Install
+
+```bash
+uv pip install -e ".[dev]"
+```
+
+## Quick Start
+
+```bash
+# List built-in agents
+agenttester agents
+
+# Run two agents on the same prompt
+agenttester run "Add unit tests for the auth module" --agents claude,aider
+
+# Use a prompt file
+agenttester run --prompt-file task.md --agents claude,codex,aider
+
+# Keep worktrees for manual inspection
+agenttester run "Refactor logging" --agents claude,aider --keep-worktrees
+```
+
+## How It Works
+
+1. You provide a prompt and select agents
+2. AgentTester creates a git worktree + branch for each agent from the current HEAD
+3. All agents run concurrently (up to 5), each in its own worktree
+4. Agent output streams to the terminal with colored prefixes
+5. A markdown comparison report is generated with diff stats and timing
+6. Worktrees are cleaned up (branches are preserved for `git diff`)
+
+Branches are named `agenttester/<run-id>/<agent-name>` so you can compare results:
+
+```bash
+git diff agenttester/a3f2c1d0/claude agenttester/a3f2c1d0/aider
+```
+
+## Configuration
+
+Copy `config.example.yaml` to `agenttester.yaml` in your target repo to customize agents. Built-in presets are available for `claude`, `aider`, and `codex`.
+
+### Command Placeholders
+
+- `{prompt}` — replaced with the shell-escaped prompt text
+- `{prompt_file}` — replaced with a path to a temp file containing the prompt
+- If neither placeholder is present, the prompt is piped to the agent via stdin
+
+### Agent Settings
+
+| Field | Description | Default |
+|-------|-------------|---------|
+| `command` | Shell command template | (required) |
+| `commit_style` | `auto` (agent commits) or `manual` (agenttester commits) | `auto` |
+| `timeout` | Max seconds before the agent is killed | `600` |
+| `env` | Extra environment variables (key-value map) | `{}` |
+
+## Interactive Model REPL
+
+For comparing responses from vLLM model servers interactively, with persistent
+conversation history within a session:
+
+```bash
+agenttester repl                        # auto-discovers agenttester.yaml
+agenttester repl --config custom.yaml   # explicit config path
+```
+
+The REPL discovers any agent in your config whose command uses `query_model.py`,
+fans out each prompt to all of them in parallel, and maintains separate
+conversation history per model. Use `/reset` to clear history or `exit` to quit.
+
+See `config.example.yaml` for example vLLM agent entries.
+
+## Development
+
+```bash
+uv pip install -e ".[dev]"
+ruff check src/
+ruff format src/
+pytest
+```
+
+## Docker
+
+```bash
+# Run against the current directory
+docker compose run --rm agenttester run "Fix the bug" --agents claude
+
+# Run against a different repo
+REPO_PATH=/path/to/repo docker compose run --rm agenttester run "Add tests" --agents claude,aider
+```
+
+## Library Usage
+
+```python
+import asyncio
+from pathlib import Path
+from rich.console import Console
+from agenttester import Orchestrator, load_config
+
+async def main():
+    agents = load_config()
+    selected = [agents["claude"], agents["aider"]]
+    orch = Orchestrator(Path(".").resolve(), Console())
+    results = await orch.run("Add unit tests", selected)
+    for r in results:
+        print(f"{r.agent_name}: exit={r.exit_code} duration={r.duration:.1f}s")
+
+asyncio.run(main())
+```

agenttester-0.1.0/config.example.yaml

@@ -0,0 +1,68 @@
+# AgentTester configuration
+# Copy to agenttester.yaml in your target repo.
+# Entries here override built-in presets (claude, aider, codex).
+#
+# host: "localhost" (default) runs locally; set to "user@server" for SSH.
+
+agents:
+  claude:
+    command: 'claude -p {prompt} --allowedTools "Bash,Read,Edit" --permission-mode acceptEdits'
+    host: localhost
+    commit_style: auto    # agent commits its own changes
+    timeout: 600
+
+  aider:
+    command: "aider --yes-always --no-auto-commits --message {prompt}"
+    host: localhost
+    commit_style: manual  # agenttester commits changes after the agent exits
+    timeout: 600
+
+  codex:
+    command: "codex exec --sandbox danger-full-access {prompt}"
+    host: localhost
+    commit_style: auto
+    timeout: 600
+
+  # Example: agent running on a remote GPU box
+  # remote-claude:
+  #   command: 'claude -p {prompt} --allowedTools "Bash,Read,Edit"'
+  #   host: user@gpu-server
+  #   remote_workdir: /home/user/agenttester
+  #   commit_style: auto
+  #   timeout: 900
+
+  # Example: custom agent with API key forwarding
+  # my-agent:
+  #   command: "my-agent --input {prompt_file} --auto"
+  #   host: localhost
+  #   commit_style: manual
+  #   timeout: 300
+  #   env:
+  #     MY_AGENT_API_KEY: "..."
+
+  # Example: agent that reads prompt from stdin (no placeholders)
+  # stdin-agent:
+  #   command: "some-agent --non-interactive"
+  #   commit_style: manual
+
+  # --- vLLM model servers on a remote EC2 host ---
+  # Deploy the host with: python setup/aws_single_host.py (in MultiAgentTrainer)
+  # Then replace HOST_IP with the public IP printed by that script.
+
+  # llama3:
+  #   command: 'agenttester query http://HOST_IP:8001 meta-llama/Meta-Llama-3-8B-Instruct {prompt}'
+  #   host: localhost
+  #   commit_style: manual
+  #   timeout: 120
+
+  # mistral:
+  #   command: 'agenttester query http://HOST_IP:8002 mistralai/Mistral-7B-Instruct-v0.2 {prompt}'
+  #   host: localhost
+  #   commit_style: manual
+  #   timeout: 120
+
+  # qwen:
+  #   command: 'agenttester query http://HOST_IP:8003 Qwen/Qwen2.5-7B-Instruct {prompt}'
+  #   host: localhost
+  #   commit_style: manual
+  #   timeout: 120

agenttester-0.1.0/deploy/Pulumi.dev.yaml

@@ -0,0 +1,7 @@
+config:
+  aws:region: us-east-1
+  agenttester-infra:instance_type: t3.large
+  agenttester-infra:instance_count: "1"
+  agenttester-infra:ssh_public_key_path: ~/.ssh/id_ed25519.pub
+  agenttester-infra:allowed_ssh_cidrs:
+    - "203.0.113.0/32"  # CHANGEME: your IP

agenttester-0.1.0/deploy/Pulumi.yaml

@@ -0,0 +1,6 @@
+name: agenttester-infra
+runtime:
+  name: python
+  options:
+    virtualenv: venv
+description: AWS infrastructure for agenttester remote agent execution

agenttester-0.1.0/deploy/README.md

@@ -0,0 +1,97 @@
+# deploy — AWS infrastructure for agenttester
+
+Pulumi project that provisions EC2 instances pre-configured to run coding agents (Claude Code, Aider, Codex) over SSH.
+
+## Prerequisites
+
+- [Pulumi CLI](https://www.pulumi.com/docs/install/)
+- AWS credentials configured (`aws configure` or env vars)
+- An SSH key pair (default: `~/.ssh/id_ed25519`)
+
+## Quick start
+
+```bash
+cd deploy
+pulumi stack init dev
+
+# Required: restrict SSH access to your IP
+pulumi config set --path 'allowed_ssh_cidrs[0]' "$(curl -s ifconfig.me)/32"
+
+pulumi up
+```
+
+## Configuration
+
+```bash
+# REQUIRED: CIDR blocks allowed to SSH into the instances
+pulumi config set --path 'allowed_ssh_cidrs[0]' "203.0.113.10/32"
+pulumi config set --path 'allowed_ssh_cidrs[1]' "10.0.0.0/8"  # e.g. VPN range
+
+# Instance type (default: t3.large)
+pulumi config set instance_type t3.xlarge
+
+# Number of instances (default: 1)
+pulumi config set instance_count 3
+
+# SSH public key path (default: ~/.ssh/id_ed25519.pub)
+pulumi config set ssh_public_key_path ~/.ssh/my_key.pub
+
+# AWS region
+pulumi config set aws:region us-west-2
+```
+
+## Outputs
+
+After `pulumi up`, the stack exports:
+
+- **`public_ips`** — IP addresses of the instances
+- **`ssh_hosts`** — Ready-to-use `ubuntu@<ip>` strings
+- **`agenttester_yaml_snippet`** — Paste directly into your `agenttester.yaml`
+
+Example:
+
+```bash
+# Get the config snippet
+pulumi stack output agenttester_yaml_snippet
+```
+
+Then add it to your target repo's `agenttester.yaml`:
+
+```yaml
+agents:
+  remote-agent-0:
+    command: 'claude -p {prompt} --allowedTools "Bash,Read,Edit"'
+    host: ubuntu@3.14.159.26
+    remote_workdir: /tmp/agenttester
+    commit_style: auto
+    timeout: 600
+```
+
+## What gets provisioned
+
+- **EC2 instances** (Ubuntu 22.04) with 50 GB gp3 root volume
+- **Security group** allowing SSH inbound from `allowed_ssh_cidrs` only, all outbound
+- **IAM role + instance profile** with SSM access (for debugging via Session Manager)
+- **Key pair** from your local SSH public key
+- **User data** that installs git, rsync, Node.js 20, Claude Code, Codex CLI, and Aider
+
+## API keys
+
+The instances don't store API keys. Forward them via the `env` field in `agenttester.yaml`:
+
+```yaml
+agents:
+  remote-claude:
+    command: 'claude -p {prompt} --allowedTools "Bash,Read,Edit"'
+    host: ubuntu@3.14.159.26
+    env:
+      ANTHROPIC_API_KEY: "sk-ant-..."
+```
+
+Or set them on the instance via SSM Parameter Store / Secrets Manager and source them in your agent command.
+
+## Teardown
+
+```bash
+pulumi destroy
+```

agenttester-0.1.0/deploy/__main__.py

@@ -0,0 +1,144 @@
+"""Pulumi program to deploy agenttester remote-execution infrastructure on AWS."""
+
+from pathlib import Path
+
+import pulumi
+import pulumi_aws as aws
+
+# ── config ────────────────────────────────────────────────────────────
+
+config = pulumi.Config()
+instance_type = config.get("instance_type") or "t3.large"
+instance_count = config.get_int("instance_count") or 1
+ssh_pub_key_path = config.get("ssh_public_key_path") or "~/.ssh/id_ed25519.pub"
+allowed_ssh_cidrs = config.require_object("allowed_ssh_cidrs")
+
+ssh_pub_key = Path(ssh_pub_key_path).expanduser().read_text().strip()
+
+# ── AMI (latest Ubuntu 22.04) ─────────────────────────────────────────
+
+ami = aws.ec2.get_ami(
+    most_recent=True,
+    owners=["099720109477"],  # Canonical
+    filters=[
+        aws.ec2.GetAmiFilterArgs(
+            name="name",
+            values=["ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-*"],
+        ),
+        aws.ec2.GetAmiFilterArgs(
+            name="virtualization-type",
+            values=["hvm"],
+        ),
+    ],
+)
+
+# ── key pair ──────────────────────────────────────────────────────────
+
+key_pair = aws.ec2.KeyPair(
+    "agenttester-key",
+    public_key=ssh_pub_key,
+)
+
+# ── security group (SSH only) ─────────────────────────────────────────
+
+sg = aws.ec2.SecurityGroup(
+    "agenttester-sg",
+    description="Allow SSH for agenttester remote agents",
+    ingress=[
+        aws.ec2.SecurityGroupIngressArgs(
+            protocol="tcp",
+            from_port=22,
+            to_port=22,
+            cidr_blocks=allowed_ssh_cidrs,
+            description="SSH",
+        ),
+    ],
+    egress=[
+        aws.ec2.SecurityGroupEgressArgs(
+            protocol="-1",
+            from_port=0,
+            to_port=0,
+            cidr_blocks=["0.0.0.0/0"],
+            description="All outbound",
+        ),
+    ],
+)
+
+# ── IAM role + instance profile ───────────────────────────────────────
+
+assume_role_policy = """{
+  "Version": "2012-10-17",
+  "Statement": [{
+    "Effect": "Allow",
+    "Principal": {"Service": "ec2.amazonaws.com"},
+    "Action": "sts:AssumeRole"
+  }]
+}"""
+
+role = aws.iam.Role(
+    "agenttester-role",
+    assume_role_policy=assume_role_policy,
+    tags={"Project": "agenttester"},
+)
+
+# SSM managed policy so you can debug via Session Manager if needed
+aws.iam.RolePolicyAttachment(
+    "agenttester-ssm",
+    role=role.name,
+    policy_arn="arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore",
+)
+
+instance_profile = aws.iam.InstanceProfile(
+    "agenttester-profile",
+    role=role.name,
+)
+
+# ── user data ─────────────────────────────────────────────────────────
+
+user_data_script = (Path(__file__).parent / "user_data.sh").read_text()
+
+# ── EC2 instances ─────────────────────────────────────────────────────
+
+instances = []
+for i in range(instance_count):
+    inst = aws.ec2.Instance(
+        f"agenttester-agent-{i}",
+        ami=ami.id,
+        instance_type=instance_type,
+        key_name=key_pair.key_name,
+        vpc_security_group_ids=[sg.id],
+        iam_instance_profile=instance_profile.name,
+        user_data=user_data_script,
+        root_block_device=aws.ec2.InstanceRootBlockDeviceArgs(
+            volume_size=50,
+            volume_type="gp3",
+        ),
+        tags={
+            "Name": f"agenttester-agent-{i}",
+            "Project": "agenttester",
+        },
+    )
+    instances.append(inst)
+
+# ── outputs ───────────────────────────────────────────────────────────
+
+pulumi.export("instance_ids", [inst.id for inst in instances])
+pulumi.export("public_ips", [inst.public_ip for inst in instances])
+pulumi.export(
+    "ssh_hosts",
+    [inst.public_ip.apply(lambda ip: f"ubuntu@{ip}") for inst in instances],
+)
+pulumi.export(
+    "agenttester_yaml_snippet",
+    pulumi.Output.all(*[inst.public_ip for inst in instances]).apply(
+        lambda ips: "\n".join(
+            f"  remote-agent-{i}:\n"
+            f'    command: \'claude -p {{prompt}} --allowedTools "Bash,Read,Edit"\'\n'
+            f"    host: ubuntu@{ip}\n"
+            f"    remote_workdir: /tmp/agenttester\n"
+            f"    commit_style: auto\n"
+            f"    timeout: 600"
+            for i, ip in enumerate(ips)
+        )
+    ),
+)

agenttester-0.1.0/deploy/requirements.txt

@@ -0,0 +1,2 @@
+pulumi>=3.0.0,<4.0.0
+pulumi-aws>=6.0.0,<7.0.0

agenttester-0.1.0/deploy/user_data.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+set -euxo pipefail
+
+# ── system packages ───────────────────────────────────────────────────
+apt-get update
+apt-get install -y git rsync curl unzip python3-pip python3-venv
+
+# ── Node.js 20 (for Claude Code and Codex CLI) ───────────────────────
+curl -fsSL https://deb.nodesource.com/setup_20.x | bash -
+apt-get install -y nodejs
+
+# ── Claude Code ───────────────────────────────────────────────────────
+npm install -g @anthropic-ai/claude-code
+
+# ── OpenAI Codex CLI ──────────────────────────────────────────────────
+npm install -g @openai/codex
+
+# ── Aider ─────────────────────────────────────────────────────────────
+pip3 install --break-system-packages aider-chat
+
+# ── workspace directory ───────────────────────────────────────────────
+mkdir -p /tmp/agenttester
+chown ubuntu:ubuntu /tmp/agenttester
+
+echo "agenttester agent host ready" > /var/log/agenttester-setup.log

agenttester-0.1.0/docker-compose.yaml

@@ -0,0 +1,14 @@
+services:
+  agenttester:
+    build: .
+    volumes:
+      - ${REPO_PATH:-.}:/repo
+      # Forward SSH agent for remote execution
+      - ${SSH_AUTH_SOCK:-/dev/null}:/ssh-agent
+      # Share host SSH config for remote hosts
+      - ~/.ssh:/root/.ssh:ro
+    environment:
+      - SSH_AUTH_SOCK=/ssh-agent
+      # Pass API keys through to agents
+      - ANTHROPIC_API_KEY
+      - OPENAI_API_KEY

agenttester-0.1.0/pyproject.toml

@@ -0,0 +1,40 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "agenttester"
+version = "0.1.0"
+description = "Send prompts to multiple coding agents in parallel and compare results"
+requires-python = ">=3.10"
+license = "MIT"
+dependencies = [
+    "typer>=0.9",
+    "pyyaml>=6.0",
+    "rich>=13.0",
+    "gitpython>=3.1",
+]
+
+[project.optional-dependencies]
+dev = [
+    "ruff>=0.4",
+    "pytest>=8.0",
+    "pytest-asyncio>=0.23",
+]
+
+[project.scripts]
+agenttester = "agenttester.cli:app"
+
+[tool.ruff]
+target-version = "py310"
+line-length = 88
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "N", "UP", "B", "SIM", "RUF"]
+
+[tool.ruff.lint.isort]
+known-first-party = ["agenttester"]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+testpaths = ["tests"]

agenttester-0.1.0/src/agenttester/__init__.py

@@ -0,0 +1,24 @@
+"""AgentTester: Multi-agent comparison tool.
+
+Usable as a CLI (``agenttester run …``), a Docker container, or a Python
+library::
+
+    from agenttester import Orchestrator, AgentConfig, load_config
+"""
+
+from .agent_runner import AgentResult, run_agent
+from .config import AgentConfig, load_config
+from .git_manager import DiffStats, GitManager
+from .orchestrator import Orchestrator
+
+__all__ = [
+    "AgentConfig",
+    "AgentResult",
+    "DiffStats",
+    "GitManager",
+    "Orchestrator",
+    "load_config",
+    "run_agent",
+]
+
+__version__ = "0.1.0"